braintrust 0.0.85 → 0.0.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +492 -477
- package/dist/cli.js +310 -385
- package/dist/index.js +544 -529
- package/dist/logger.d.ts +143 -138
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
package/dist/browser.js
CHANGED
|
@@ -70,7 +70,7 @@ function v4(options, buf, offset) {
|
|
|
70
70
|
}
|
|
71
71
|
var v4_default = v4;
|
|
72
72
|
|
|
73
|
-
//
|
|
73
|
+
// ../core/js/dist/index.mjs
|
|
74
74
|
var TRANSACTION_ID_FIELD = "_xact_id";
|
|
75
75
|
var IS_MERGE_FIELD = "_is_merge";
|
|
76
76
|
function mergeDicts(mergeInto, mergeFrom) {
|
|
@@ -159,12 +159,12 @@ var NoopSpan = class {
|
|
|
159
159
|
}
|
|
160
160
|
log(_) {
|
|
161
161
|
}
|
|
162
|
-
|
|
163
|
-
return this;
|
|
164
|
-
}
|
|
165
|
-
traced(_0, callback, _1) {
|
|
162
|
+
traced(callback, _1) {
|
|
166
163
|
return callback(this);
|
|
167
164
|
}
|
|
165
|
+
startSpan(_1) {
|
|
166
|
+
return this;
|
|
167
|
+
}
|
|
168
168
|
end(args) {
|
|
169
169
|
return args?.endTime ?? getCurrentUnixTimestamp();
|
|
170
170
|
}
|
|
@@ -172,16 +172,25 @@ var NoopSpan = class {
|
|
|
172
172
|
return this.end(args);
|
|
173
173
|
}
|
|
174
174
|
};
|
|
175
|
-
var
|
|
175
|
+
var NOOP_SPAN = new NoopSpan();
|
|
176
176
|
var BraintrustState = class {
|
|
177
177
|
constructor() {
|
|
178
|
+
this.apiUrl = null;
|
|
179
|
+
this.loginToken = null;
|
|
180
|
+
this.orgId = null;
|
|
181
|
+
this.orgName = null;
|
|
182
|
+
this.logUrl = null;
|
|
183
|
+
this.loggedIn = false;
|
|
184
|
+
this._apiConn = null;
|
|
185
|
+
this._logConn = null;
|
|
178
186
|
this.id = v4_default();
|
|
179
|
-
this.currentExperiment =
|
|
180
|
-
this.currentLogger =
|
|
187
|
+
this.currentExperiment = void 0;
|
|
188
|
+
this.currentLogger = void 0;
|
|
181
189
|
this.currentSpan = isomorph_default.newAsyncLocalStorage();
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
190
|
+
this.resetLoginInfo();
|
|
191
|
+
globalThis.__inherited_braintrust_state = this;
|
|
192
|
+
}
|
|
193
|
+
resetLoginInfo() {
|
|
185
194
|
this.apiUrl = null;
|
|
186
195
|
this.loginToken = null;
|
|
187
196
|
this.orgId = null;
|
|
@@ -190,7 +199,6 @@ var BraintrustState = class {
|
|
|
190
199
|
this.loggedIn = false;
|
|
191
200
|
this._apiConn = null;
|
|
192
201
|
this._logConn = null;
|
|
193
|
-
globalThis.__inherited_braintrust_state = this;
|
|
194
202
|
}
|
|
195
203
|
apiConn() {
|
|
196
204
|
if (!this._apiConn) {
|
|
@@ -219,36 +227,6 @@ function _internalSetInitialState() {
|
|
|
219
227
|
_state = globalThis.__inherited_braintrust_state || new BraintrustState();
|
|
220
228
|
}
|
|
221
229
|
var _internalGetGlobalState = () => _state;
|
|
222
|
-
var UnterminatedObjectsHandler = class {
|
|
223
|
-
constructor() {
|
|
224
|
-
this.unterminatedObjects = /* @__PURE__ */ new Map();
|
|
225
|
-
isomorph_default.processOn("exit", () => {
|
|
226
|
-
this.warnUnterminated();
|
|
227
|
-
});
|
|
228
|
-
}
|
|
229
|
-
addUnterminated(obj, createdLocation) {
|
|
230
|
-
this.unterminatedObjects.set(obj, createdLocation);
|
|
231
|
-
}
|
|
232
|
-
removeUnterminated(obj) {
|
|
233
|
-
this.unterminatedObjects.delete(obj);
|
|
234
|
-
}
|
|
235
|
-
warnUnterminated() {
|
|
236
|
-
if (this.unterminatedObjects.size === 0) {
|
|
237
|
-
return;
|
|
238
|
-
}
|
|
239
|
-
let warningMessage = "WARNING: Did not close the following braintrust objects. We recommend running `.close` on the listed objects, or by running them inside a callback so they are closed automatically:";
|
|
240
|
-
this.unterminatedObjects.forEach((createdLocation, obj) => {
|
|
241
|
-
let msg = `
|
|
242
|
-
Object of type ${obj?.constructor?.name}`;
|
|
243
|
-
if (createdLocation) {
|
|
244
|
-
msg += ` created at ${JSON.stringify(createdLocation)}`;
|
|
245
|
-
}
|
|
246
|
-
warningMessage += msg;
|
|
247
|
-
});
|
|
248
|
-
console.warn(warningMessage);
|
|
249
|
-
}
|
|
250
|
-
};
|
|
251
|
-
var unterminatedObjects = new UnterminatedObjectsHandler();
|
|
252
230
|
var FailedHTTPResponse = class extends Error {
|
|
253
231
|
constructor(status, text, data = null) {
|
|
254
232
|
super(`${status}: ${text}`);
|
|
@@ -355,45 +333,29 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
355
333
|
return await resp.json();
|
|
356
334
|
}
|
|
357
335
|
};
|
|
358
|
-
var Project = class {
|
|
359
|
-
constructor({ name, id }) {
|
|
360
|
-
this.name = name;
|
|
361
|
-
this.id = id;
|
|
362
|
-
}
|
|
363
|
-
async lazyInit() {
|
|
364
|
-
if (this.id === void 0) {
|
|
365
|
-
const response = await _state.apiConn().post_json("api/project/register", {
|
|
366
|
-
project_name: this.name || GLOBAL_PROJECT,
|
|
367
|
-
org_id: _state.orgId
|
|
368
|
-
});
|
|
369
|
-
this.id = response.project.id;
|
|
370
|
-
this.name = response.project.name;
|
|
371
|
-
} else if (this.name === void 0) {
|
|
372
|
-
const response = await _state.apiConn().get_json("api/project", {
|
|
373
|
-
id: this.id
|
|
374
|
-
});
|
|
375
|
-
this.name = response.name;
|
|
376
|
-
}
|
|
377
|
-
return { id: this.id, name: this.name };
|
|
378
|
-
}
|
|
379
|
-
};
|
|
380
336
|
var Logger = class {
|
|
381
|
-
constructor(
|
|
382
|
-
this.loggedIn = false;
|
|
337
|
+
constructor(lazyMetadata, logOptions = {}) {
|
|
383
338
|
// For type identification.
|
|
384
339
|
this.kind = "logger";
|
|
385
|
-
this.
|
|
386
|
-
this.lazyProject = lazyProject;
|
|
340
|
+
this.lazyMetadata = lazyMetadata;
|
|
387
341
|
this.logOptions = logOptions;
|
|
388
|
-
this.
|
|
342
|
+
const logConn = this.getState().then((state) => state.logConn());
|
|
343
|
+
this.bgLogger = new BackgroundLogger(logConn);
|
|
389
344
|
this.lastStartTime = getCurrentUnixTimestamp();
|
|
390
345
|
}
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
await this.
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
346
|
+
get org_id() {
|
|
347
|
+
return (async () => {
|
|
348
|
+
return (await this.lazyMetadata).org_id;
|
|
349
|
+
})();
|
|
350
|
+
}
|
|
351
|
+
get project() {
|
|
352
|
+
return (async () => {
|
|
353
|
+
return (await this.lazyMetadata).project;
|
|
354
|
+
})();
|
|
355
|
+
}
|
|
356
|
+
async getState() {
|
|
357
|
+
await this.lazyMetadata;
|
|
358
|
+
return _state;
|
|
397
359
|
}
|
|
398
360
|
/**
|
|
399
361
|
* Log a single event. The event will be batched and uploaded behind the scenes if `logOptions.asyncFlush` is true.
|
|
@@ -408,54 +370,87 @@ var Logger = class {
|
|
|
408
370
|
* @param event.id: (Optional) a unique identifier for the event. If you don't provide one, BrainTrust will generate one for you.
|
|
409
371
|
* :returns: The `id` of the logged event.
|
|
410
372
|
*/
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
const span = await this.startSpan({ startTime: this.lastStartTime, event });
|
|
373
|
+
log(event) {
|
|
374
|
+
const span = this.startSpan({ startTime: this.lastStartTime, event });
|
|
414
375
|
this.lastStartTime = span.end();
|
|
415
|
-
|
|
416
|
-
|
|
376
|
+
const ret = span.id;
|
|
377
|
+
if (this.logOptions.asyncFlush === true) {
|
|
378
|
+
return ret;
|
|
379
|
+
} else {
|
|
380
|
+
return (async () => {
|
|
381
|
+
await this.flush();
|
|
382
|
+
return ret;
|
|
383
|
+
})();
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* Create a new toplevel span underneath the logger. The name defaults to "root".
|
|
388
|
+
*
|
|
389
|
+
* See `Span.traced` for full details.
|
|
390
|
+
*/
|
|
391
|
+
traced(callback, args) {
|
|
392
|
+
const { setCurrent, ...argsRest } = args ?? {};
|
|
393
|
+
const span = this.startSpan(argsRest);
|
|
394
|
+
const ret = runFinally(
|
|
395
|
+
() => {
|
|
396
|
+
if (setCurrent ?? true) {
|
|
397
|
+
return withCurrent(span, callback);
|
|
398
|
+
} else {
|
|
399
|
+
return callback(span);
|
|
400
|
+
}
|
|
401
|
+
},
|
|
402
|
+
() => span.end()
|
|
403
|
+
);
|
|
404
|
+
if (this.logOptions.asyncFlush) {
|
|
405
|
+
return ret;
|
|
406
|
+
} else {
|
|
407
|
+
return (async () => {
|
|
408
|
+
const awaitedRet = await ret;
|
|
409
|
+
await this.flush();
|
|
410
|
+
return awaitedRet;
|
|
411
|
+
})();
|
|
417
412
|
}
|
|
418
|
-
return span.id;
|
|
419
413
|
}
|
|
420
414
|
/**
|
|
421
|
-
*
|
|
415
|
+
* Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
|
|
422
416
|
*
|
|
423
|
-
* See `
|
|
417
|
+
* See `traced` for full details.
|
|
424
418
|
*/
|
|
425
|
-
|
|
426
|
-
await this.lazyLogin();
|
|
427
|
-
const project = await this.lazyProject.lazyInit();
|
|
419
|
+
startSpan(args) {
|
|
428
420
|
const { name, ...argsRest } = args ?? {};
|
|
421
|
+
const parentIds = (async () => ({
|
|
422
|
+
kind: "project_log",
|
|
423
|
+
org_id: await this.org_id,
|
|
424
|
+
project_id: (await this.project).id,
|
|
425
|
+
log_id: "g"
|
|
426
|
+
}))();
|
|
429
427
|
return new SpanImpl({
|
|
428
|
+
parentIds,
|
|
430
429
|
bgLogger: this.bgLogger,
|
|
431
430
|
name: name ?? "root",
|
|
432
|
-
...argsRest
|
|
433
|
-
rootProject: project
|
|
431
|
+
...argsRest
|
|
434
432
|
});
|
|
435
433
|
}
|
|
436
|
-
/**
|
|
437
|
-
* Wrapper over `Logger.startSpan`, which passes the initialized `Span` it to the given callback and ends it afterwards. See `Span.traced` for full details.
|
|
438
|
-
*/
|
|
439
|
-
async traced(callback, args) {
|
|
440
|
-
const { setCurrent, ...argsRest } = args ?? {};
|
|
441
|
-
const span = await this.startSpan(argsRest);
|
|
442
|
-
try {
|
|
443
|
-
let ret = null;
|
|
444
|
-
return await (setCurrent ?? true ? withCurrent(span, () => callback(span)) : callback(span));
|
|
445
|
-
} finally {
|
|
446
|
-
span.end();
|
|
447
|
-
if (!this.logOptions.asyncFlush) {
|
|
448
|
-
await this.flush();
|
|
449
|
-
}
|
|
450
|
-
}
|
|
451
|
-
}
|
|
452
434
|
/*
|
|
453
435
|
* Flush any pending logs to the server.
|
|
454
436
|
*/
|
|
455
437
|
async flush() {
|
|
456
438
|
return await this.bgLogger.flush();
|
|
457
439
|
}
|
|
440
|
+
get asyncFlush() {
|
|
441
|
+
return this.logOptions.asyncFlush;
|
|
442
|
+
}
|
|
458
443
|
};
|
|
444
|
+
function castLogger(logger, asyncFlush) {
|
|
445
|
+
if (logger === void 0)
|
|
446
|
+
return void 0;
|
|
447
|
+
if (asyncFlush && !!asyncFlush !== !!logger.asyncFlush) {
|
|
448
|
+
throw new Error(
|
|
449
|
+
`Asserted asyncFlush setting ${asyncFlush} does not match stored logger's setting ${logger.asyncFlush}`
|
|
450
|
+
);
|
|
451
|
+
}
|
|
452
|
+
return logger;
|
|
453
|
+
}
|
|
459
454
|
var MaxRequestSize = 6 * 1024 * 1024;
|
|
460
455
|
function constructJsonArray(items) {
|
|
461
456
|
return `[${items.join(",")}]`;
|
|
@@ -466,10 +461,11 @@ function now() {
|
|
|
466
461
|
return (/* @__PURE__ */ new Date()).getTime();
|
|
467
462
|
}
|
|
468
463
|
var BackgroundLogger = class {
|
|
469
|
-
constructor() {
|
|
464
|
+
constructor(logConn) {
|
|
470
465
|
this.items = [];
|
|
471
466
|
this.active_flush = Promise.resolve([]);
|
|
472
467
|
this.active_flush_resolved = true;
|
|
468
|
+
this.logConn = logConn;
|
|
473
469
|
isomorph_default.processOn("beforeExit", async () => {
|
|
474
470
|
await this.flush();
|
|
475
471
|
});
|
|
@@ -483,8 +479,9 @@ var BackgroundLogger = class {
|
|
|
483
479
|
}
|
|
484
480
|
async flush_once(batchSize = DefaultBatchSize) {
|
|
485
481
|
this.active_flush_resolved = false;
|
|
486
|
-
const
|
|
482
|
+
const itemPromises = this.items;
|
|
487
483
|
this.items = [];
|
|
484
|
+
const allItems = mergeRowBatch(await Promise.all(itemPromises)).reverse();
|
|
488
485
|
let postPromises = [];
|
|
489
486
|
while (true) {
|
|
490
487
|
const items = [];
|
|
@@ -509,7 +506,7 @@ var BackgroundLogger = class {
|
|
|
509
506
|
for (let i = 0; i < NumRetries; i++) {
|
|
510
507
|
const startTime = now();
|
|
511
508
|
try {
|
|
512
|
-
return (await
|
|
509
|
+
return (await (await this.logConn).post_json("logs", itemsS)).map(
|
|
513
510
|
(res) => res.id
|
|
514
511
|
);
|
|
515
512
|
} catch (e) {
|
|
@@ -550,7 +547,7 @@ var BackgroundLogger = class {
|
|
|
550
547
|
}
|
|
551
548
|
}
|
|
552
549
|
};
|
|
553
|
-
|
|
550
|
+
function init(project, options = {}) {
|
|
554
551
|
const {
|
|
555
552
|
experiment,
|
|
556
553
|
description,
|
|
@@ -561,79 +558,129 @@ async function init(project, options = {}) {
|
|
|
561
558
|
apiUrl,
|
|
562
559
|
apiKey,
|
|
563
560
|
orgName,
|
|
564
|
-
disableCache,
|
|
565
561
|
metadata
|
|
566
562
|
} = options || {};
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
()
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
563
|
+
const lazyMetadata = (async () => {
|
|
564
|
+
await login({
|
|
565
|
+
orgName,
|
|
566
|
+
apiKey,
|
|
567
|
+
apiUrl
|
|
568
|
+
});
|
|
569
|
+
const args = {
|
|
570
|
+
project_name: project,
|
|
571
|
+
org_id: _state.orgId
|
|
572
|
+
};
|
|
573
|
+
if (experiment) {
|
|
574
|
+
args["experiment_name"] = experiment;
|
|
575
|
+
}
|
|
576
|
+
if (description) {
|
|
577
|
+
args["description"] = description;
|
|
578
|
+
}
|
|
579
|
+
if (update) {
|
|
580
|
+
args["update"] = update;
|
|
581
|
+
}
|
|
582
|
+
const repoStatus = await isomorph_default.getRepoStatus();
|
|
583
|
+
if (repoStatus) {
|
|
584
|
+
args["repo_info"] = repoStatus;
|
|
585
|
+
}
|
|
586
|
+
if (baseExperiment) {
|
|
587
|
+
args["base_experiment"] = baseExperiment;
|
|
588
|
+
} else {
|
|
589
|
+
args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
|
|
590
|
+
}
|
|
591
|
+
if (dataset !== void 0) {
|
|
592
|
+
args["dataset_id"] = dataset.id;
|
|
593
|
+
args["dataset_version"] = await dataset.version();
|
|
594
|
+
}
|
|
595
|
+
if (isPublic !== void 0) {
|
|
596
|
+
args["public"] = isPublic;
|
|
597
|
+
}
|
|
598
|
+
if (metadata) {
|
|
599
|
+
args["metadata"] = metadata;
|
|
600
|
+
}
|
|
601
|
+
let response = null;
|
|
602
|
+
while (true) {
|
|
603
|
+
try {
|
|
604
|
+
response = await _state.apiConn().post_json("api/experiment/register", args);
|
|
605
|
+
break;
|
|
606
|
+
} catch (e) {
|
|
607
|
+
if (args["base_experiment"] && `${"data" in e && e.data}`.includes("base experiment")) {
|
|
608
|
+
console.warn(`Base experiment ${args["base_experiment"]} not found.`);
|
|
609
|
+
delete args["base_experiment"];
|
|
610
|
+
} else {
|
|
611
|
+
throw e;
|
|
612
|
+
}
|
|
591
613
|
}
|
|
592
|
-
}
|
|
593
|
-
|
|
614
|
+
}
|
|
615
|
+
return {
|
|
616
|
+
project: {
|
|
617
|
+
id: response.project.id,
|
|
618
|
+
name: response.project.name,
|
|
619
|
+
fullInfo: response.project
|
|
620
|
+
},
|
|
621
|
+
experiment: {
|
|
622
|
+
id: response.experiment.id,
|
|
623
|
+
name: response.experiment.name,
|
|
624
|
+
fullInfo: response.experiment
|
|
625
|
+
}
|
|
626
|
+
};
|
|
627
|
+
})();
|
|
628
|
+
const ret = new Experiment(lazyMetadata, dataset);
|
|
629
|
+
if (options.setCurrent ?? true) {
|
|
630
|
+
_state.currentExperiment = ret;
|
|
631
|
+
}
|
|
632
|
+
return ret;
|
|
633
|
+
}
|
|
634
|
+
function withExperiment(project, callback, options = {}) {
|
|
635
|
+
console.warn(
|
|
636
|
+
"withExperiment is deprecated and will be removed in a future version of braintrust. Simply create the experiment with `init`."
|
|
594
637
|
);
|
|
638
|
+
const experiment = init(project, options);
|
|
639
|
+
return callback(experiment);
|
|
595
640
|
}
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
() => {
|
|
600
|
-
if (options.setCurrent ?? true) {
|
|
601
|
-
return withCurrent(logger, () => callback(logger));
|
|
602
|
-
} else {
|
|
603
|
-
return callback(logger);
|
|
604
|
-
}
|
|
605
|
-
},
|
|
606
|
-
() => logger.flush()
|
|
641
|
+
function withLogger(callback, options = {}) {
|
|
642
|
+
console.warn(
|
|
643
|
+
"withLogger is deprecated and will be removed in a future version of braintrust. Simply create the logger with `initLogger`."
|
|
607
644
|
);
|
|
645
|
+
const logger = initLogger(options);
|
|
646
|
+
return callback(logger);
|
|
608
647
|
}
|
|
609
|
-
|
|
610
|
-
const {
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
648
|
+
function initDataset(project, options = {}) {
|
|
649
|
+
const { dataset, description, version, apiUrl, apiKey, orgName } = options || {};
|
|
650
|
+
const lazyMetadata = (async () => {
|
|
651
|
+
await login({
|
|
652
|
+
orgName,
|
|
653
|
+
apiKey,
|
|
654
|
+
apiUrl
|
|
655
|
+
});
|
|
656
|
+
const args = {
|
|
657
|
+
org_id: _state.orgId,
|
|
658
|
+
project_name: project,
|
|
659
|
+
dataset_name: dataset,
|
|
660
|
+
description
|
|
661
|
+
};
|
|
662
|
+
const response = await _state.apiConn().post_json("api/dataset/register", args);
|
|
663
|
+
return {
|
|
664
|
+
project: {
|
|
665
|
+
id: response.project.id,
|
|
666
|
+
name: response.project.name,
|
|
667
|
+
fullInfo: response.project
|
|
668
|
+
},
|
|
669
|
+
dataset: {
|
|
670
|
+
id: response.dataset.id,
|
|
671
|
+
name: response.dataset.name,
|
|
672
|
+
fullInfo: response.dataset
|
|
673
|
+
}
|
|
674
|
+
};
|
|
675
|
+
})();
|
|
676
|
+
return new Dataset(lazyMetadata, version);
|
|
630
677
|
}
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
() => callback(dataset),
|
|
635
|
-
() => dataset.close()
|
|
678
|
+
function withDataset(project, callback, options = {}) {
|
|
679
|
+
console.warn(
|
|
680
|
+
"withDataset is deprecated and will be removed in a future version of braintrust. Simply create the dataset with `initDataset`."
|
|
636
681
|
);
|
|
682
|
+
const dataset = initDataset(project, options);
|
|
683
|
+
return callback(dataset);
|
|
637
684
|
}
|
|
638
685
|
function initLogger(options = {}) {
|
|
639
686
|
const {
|
|
@@ -643,42 +690,67 @@ function initLogger(options = {}) {
|
|
|
643
690
|
apiUrl,
|
|
644
691
|
apiKey,
|
|
645
692
|
orgName,
|
|
646
|
-
|
|
693
|
+
forceLogin
|
|
647
694
|
} = options || {};
|
|
648
|
-
const
|
|
695
|
+
const lazyMetadata = (async () => {
|
|
649
696
|
await login({
|
|
650
697
|
orgName,
|
|
651
|
-
disableCache,
|
|
652
698
|
apiKey,
|
|
653
|
-
apiUrl
|
|
699
|
+
apiUrl,
|
|
700
|
+
forceLogin
|
|
654
701
|
});
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
702
|
+
const org_id = _state.orgId;
|
|
703
|
+
if (projectId === void 0) {
|
|
704
|
+
const response = await _state.apiConn().post_json("api/project/register", {
|
|
705
|
+
project_name: projectName || GLOBAL_PROJECT,
|
|
706
|
+
org_id
|
|
707
|
+
});
|
|
708
|
+
return {
|
|
709
|
+
org_id,
|
|
710
|
+
project: {
|
|
711
|
+
id: response.project.id,
|
|
712
|
+
name: response.project.name,
|
|
713
|
+
fullInfo: response.project
|
|
714
|
+
}
|
|
715
|
+
};
|
|
716
|
+
} else if (projectName === void 0) {
|
|
717
|
+
const response = await _state.apiConn().get_json("api/project", {
|
|
718
|
+
id: projectId
|
|
719
|
+
});
|
|
720
|
+
return {
|
|
721
|
+
org_id,
|
|
722
|
+
project: {
|
|
723
|
+
id: projectId,
|
|
724
|
+
name: response.name,
|
|
725
|
+
fullInfo: response.project
|
|
726
|
+
}
|
|
727
|
+
};
|
|
728
|
+
} else {
|
|
729
|
+
return {
|
|
730
|
+
org_id,
|
|
731
|
+
project: { id: projectId, name: projectName, fullInfo: {} }
|
|
732
|
+
};
|
|
664
733
|
}
|
|
665
|
-
);
|
|
734
|
+
})();
|
|
735
|
+
const ret = new Logger(lazyMetadata, {
|
|
736
|
+
asyncFlush
|
|
737
|
+
});
|
|
738
|
+
if (options.setCurrent ?? true) {
|
|
739
|
+
_state.currentLogger = ret;
|
|
740
|
+
}
|
|
741
|
+
return ret;
|
|
666
742
|
}
|
|
667
743
|
async function login(options = {}) {
|
|
668
744
|
const {
|
|
669
745
|
apiUrl = isomorph_default.getEnv("BRAINTRUST_API_URL") || "https://www.braintrustdata.com",
|
|
670
746
|
apiKey = isomorph_default.getEnv("BRAINTRUST_API_KEY"),
|
|
671
|
-
orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME")
|
|
672
|
-
disableCache = false
|
|
747
|
+
orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME")
|
|
673
748
|
} = options || {};
|
|
674
749
|
let { forceLogin = false } = options || {};
|
|
675
|
-
if (apiUrl != _state.apiUrl || apiKey !== void 0 && HTTPConnection.sanitize_token(apiKey) != _state.loginToken || orgName !== void 0 && orgName != _state.orgName) {
|
|
676
|
-
forceLogin = true;
|
|
677
|
-
}
|
|
678
750
|
if (_state.loggedIn && !forceLogin) {
|
|
679
751
|
return;
|
|
680
752
|
}
|
|
681
|
-
_state
|
|
753
|
+
_state.resetLoginInfo();
|
|
682
754
|
_state.apiUrl = apiUrl;
|
|
683
755
|
let conn = null;
|
|
684
756
|
if (apiKey !== void 0) {
|
|
@@ -711,78 +783,87 @@ async function login(options = {}) {
|
|
|
711
783
|
_state.loggedIn = true;
|
|
712
784
|
}
|
|
713
785
|
function log(event) {
|
|
714
|
-
|
|
715
|
-
|
|
786
|
+
console.warn(
|
|
787
|
+
"braintrust.log is deprecated and will be removed in a future version of braintrust. Use `experiment.log` instead."
|
|
788
|
+
);
|
|
789
|
+
const e = currentExperiment();
|
|
790
|
+
if (!e) {
|
|
716
791
|
throw new Error("Not initialized. Please call init() first");
|
|
717
792
|
}
|
|
718
|
-
return
|
|
793
|
+
return e.log(event);
|
|
719
794
|
}
|
|
720
795
|
async function summarize(options = {}) {
|
|
721
|
-
|
|
722
|
-
|
|
796
|
+
console.warn(
|
|
797
|
+
"braintrust.summarize is deprecated and will be removed in a future version of braintrust. Use `experiment.summarize` instead."
|
|
798
|
+
);
|
|
799
|
+
const e = currentExperiment();
|
|
800
|
+
if (!e) {
|
|
723
801
|
throw new Error("Not initialized. Please call init() first");
|
|
724
802
|
}
|
|
725
|
-
return await
|
|
803
|
+
return await e.summarize(options);
|
|
726
804
|
}
|
|
727
805
|
function currentExperiment() {
|
|
728
|
-
return _state.currentExperiment
|
|
806
|
+
return _state.currentExperiment;
|
|
729
807
|
}
|
|
730
|
-
function currentLogger() {
|
|
731
|
-
return _state.currentLogger
|
|
808
|
+
function currentLogger(options) {
|
|
809
|
+
return castLogger(_state.currentLogger, options?.asyncFlush);
|
|
732
810
|
}
|
|
733
811
|
function currentSpan() {
|
|
734
|
-
return _state.currentSpan.getStore() ??
|
|
812
|
+
return _state.currentSpan.getStore() ?? NOOP_SPAN;
|
|
735
813
|
}
|
|
736
|
-
function
|
|
737
|
-
const { name: nameOpt, ...argsRest } = args ?? {};
|
|
738
|
-
const name = (nameOpt ?? isomorph_default.getCallerLocation()?.caller_functionname) || "root";
|
|
814
|
+
function getSpanParentObject(options) {
|
|
739
815
|
const parentSpan = currentSpan();
|
|
740
|
-
if (!parentSpan) {
|
|
741
|
-
|
|
742
|
-
"Cannot call startSpan() from outside a trace. Please wrap this code in a traced() callback."
|
|
743
|
-
);
|
|
744
|
-
}
|
|
745
|
-
if (!Object.is(parentSpan, noopSpan)) {
|
|
746
|
-
return parentSpan.startSpan(name, argsRest);
|
|
816
|
+
if (!Object.is(parentSpan, NOOP_SPAN)) {
|
|
817
|
+
return parentSpan;
|
|
747
818
|
}
|
|
748
819
|
const experiment = currentExperiment();
|
|
749
820
|
if (experiment) {
|
|
750
|
-
return experiment
|
|
821
|
+
return experiment;
|
|
751
822
|
}
|
|
752
|
-
const logger = currentLogger();
|
|
823
|
+
const logger = currentLogger(options);
|
|
753
824
|
if (logger) {
|
|
754
|
-
|
|
755
|
-
"Cannot start a span within a logger from startSpan(). Use logger.startSpan() instead."
|
|
756
|
-
);
|
|
825
|
+
return logger;
|
|
757
826
|
}
|
|
758
|
-
return
|
|
827
|
+
return NOOP_SPAN;
|
|
759
828
|
}
|
|
760
829
|
function traced(callback, args) {
|
|
761
|
-
const span =
|
|
762
|
-
|
|
830
|
+
const { span, parentObject } = startSpanReturnParent(args);
|
|
831
|
+
const ret = runFinally(
|
|
763
832
|
() => {
|
|
764
833
|
if (args?.setCurrent ?? true) {
|
|
765
|
-
return withCurrent(span,
|
|
834
|
+
return withCurrent(span, callback);
|
|
766
835
|
} else {
|
|
767
836
|
return callback(span);
|
|
768
837
|
}
|
|
769
838
|
},
|
|
770
839
|
() => span.end()
|
|
771
840
|
);
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
if (object.kind === "experiment") {
|
|
775
|
-
return _state.currentExperiment.run(object, callback);
|
|
776
|
-
} else if (object.kind === "logger") {
|
|
777
|
-
return _state.currentLogger.run(object, callback);
|
|
778
|
-
} else if (object.kind === "span") {
|
|
779
|
-
return _state.currentSpan.run(object, callback);
|
|
841
|
+
if (args?.asyncFlush) {
|
|
842
|
+
return ret;
|
|
780
843
|
} else {
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
844
|
+
return (async () => {
|
|
845
|
+
const awaitedRet = await ret;
|
|
846
|
+
if (parentObject.kind === "logger") {
|
|
847
|
+
await parentObject.flush();
|
|
848
|
+
}
|
|
849
|
+
return awaitedRet;
|
|
850
|
+
})();
|
|
784
851
|
}
|
|
785
852
|
}
|
|
853
|
+
function startSpan(args) {
|
|
854
|
+
return startSpanReturnParent(args).span;
|
|
855
|
+
}
|
|
856
|
+
function startSpanReturnParent(args) {
|
|
857
|
+
const parentObject = getSpanParentObject({
|
|
858
|
+
asyncFlush: args?.asyncFlush
|
|
859
|
+
});
|
|
860
|
+
const { name: nameOpt, ...argsRest } = args ?? {};
|
|
861
|
+
const name = parentObject.kind === "span" ? nameOpt : nameOpt ?? "root";
|
|
862
|
+
return { span: parentObject.startSpan({ name, ...argsRest }), parentObject };
|
|
863
|
+
}
|
|
864
|
+
function withCurrent(span, callback) {
|
|
865
|
+
return _state.currentSpan.run(span, () => callback(span));
|
|
866
|
+
}
|
|
786
867
|
function _check_org_info(org_info, org_name) {
|
|
787
868
|
if (org_info.length === 0) {
|
|
788
869
|
throw new Error("This user is not part of any organizations.");
|
|
@@ -869,83 +950,34 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
|
|
|
869
950
|
}
|
|
870
951
|
return event;
|
|
871
952
|
}
|
|
872
|
-
async function _initExperiment(projectName, {
|
|
873
|
-
experimentName,
|
|
874
|
-
description,
|
|
875
|
-
dataset,
|
|
876
|
-
update,
|
|
877
|
-
baseExperiment,
|
|
878
|
-
isPublic,
|
|
879
|
-
metadata
|
|
880
|
-
} = {
|
|
881
|
-
experimentName: void 0,
|
|
882
|
-
description: void 0,
|
|
883
|
-
baseExperiment: void 0,
|
|
884
|
-
isPublic: false,
|
|
885
|
-
metadata: void 0
|
|
886
|
-
}) {
|
|
887
|
-
const args = {
|
|
888
|
-
project_name: projectName,
|
|
889
|
-
org_id: _state.orgId
|
|
890
|
-
};
|
|
891
|
-
if (experimentName) {
|
|
892
|
-
args["experiment_name"] = experimentName;
|
|
893
|
-
}
|
|
894
|
-
if (description) {
|
|
895
|
-
args["description"] = description;
|
|
896
|
-
}
|
|
897
|
-
if (update) {
|
|
898
|
-
args["update"] = update;
|
|
899
|
-
}
|
|
900
|
-
const repoStatus = await isomorph_default.getRepoStatus();
|
|
901
|
-
if (repoStatus) {
|
|
902
|
-
args["repo_info"] = repoStatus;
|
|
903
|
-
}
|
|
904
|
-
if (baseExperiment) {
|
|
905
|
-
args["base_experiment"] = baseExperiment;
|
|
906
|
-
} else {
|
|
907
|
-
args["ancestor_commits"] = await isomorph_default.getPastNAncestors();
|
|
908
|
-
}
|
|
909
|
-
if (dataset !== void 0) {
|
|
910
|
-
args["dataset_id"] = dataset.id;
|
|
911
|
-
args["dataset_version"] = await dataset.version();
|
|
912
|
-
}
|
|
913
|
-
if (isPublic !== void 0) {
|
|
914
|
-
args["public"] = isPublic;
|
|
915
|
-
}
|
|
916
|
-
if (metadata) {
|
|
917
|
-
args["metadata"] = metadata;
|
|
918
|
-
}
|
|
919
|
-
let response = null;
|
|
920
|
-
while (true) {
|
|
921
|
-
try {
|
|
922
|
-
response = await _state.apiConn().post_json("api/experiment/register", args);
|
|
923
|
-
break;
|
|
924
|
-
} catch (e) {
|
|
925
|
-
if (args["base_experiment"] && `${"data" in e && e.data}`.includes("base experiment")) {
|
|
926
|
-
console.warn(`Base experiment ${args["base_experiment"]} not found.`);
|
|
927
|
-
delete args["base_experiment"];
|
|
928
|
-
} else {
|
|
929
|
-
throw e;
|
|
930
|
-
}
|
|
931
|
-
}
|
|
932
|
-
}
|
|
933
|
-
const project = response.project;
|
|
934
|
-
const experiment = response.experiment;
|
|
935
|
-
return new Experiment(project, experiment.id, experiment.name, dataset);
|
|
936
|
-
}
|
|
937
953
|
var Experiment = class {
|
|
938
|
-
constructor(
|
|
954
|
+
constructor(lazyMetadata, dataset) {
|
|
939
955
|
// For type identification.
|
|
940
956
|
this.kind = "experiment";
|
|
941
|
-
this.
|
|
942
|
-
this.project = project;
|
|
943
|
-
this.id = id;
|
|
944
|
-
this.name = name;
|
|
957
|
+
this.lazyMetadata = lazyMetadata;
|
|
945
958
|
this.dataset = dataset;
|
|
946
|
-
this.
|
|
959
|
+
const logConn = this.getState().then((state) => state.logConn());
|
|
960
|
+
this.bgLogger = new BackgroundLogger(logConn);
|
|
947
961
|
this.lastStartTime = getCurrentUnixTimestamp();
|
|
948
|
-
|
|
962
|
+
}
|
|
963
|
+
get id() {
|
|
964
|
+
return (async () => {
|
|
965
|
+
return (await this.lazyMetadata).experiment.id;
|
|
966
|
+
})();
|
|
967
|
+
}
|
|
968
|
+
get name() {
|
|
969
|
+
return (async () => {
|
|
970
|
+
return (await this.lazyMetadata).experiment.name;
|
|
971
|
+
})();
|
|
972
|
+
}
|
|
973
|
+
get project() {
|
|
974
|
+
return (async () => {
|
|
975
|
+
return (await this.lazyMetadata).project;
|
|
976
|
+
})();
|
|
977
|
+
}
|
|
978
|
+
async getState() {
|
|
979
|
+
await this.lazyMetadata;
|
|
980
|
+
return _state;
|
|
949
981
|
}
|
|
950
982
|
/**
|
|
951
983
|
* Log a single event to the experiment. The event will be batched and uploaded behind the scenes.
|
|
@@ -963,29 +995,15 @@ var Experiment = class {
|
|
|
963
995
|
* :returns: The `id` of the logged event.
|
|
964
996
|
*/
|
|
965
997
|
log(event) {
|
|
966
|
-
this.checkNotFinished();
|
|
967
998
|
event = validateAndSanitizeExperimentLogFullArgs(event, !!this.dataset);
|
|
968
999
|
const span = this.startSpan({ startTime: this.lastStartTime, event });
|
|
969
1000
|
this.lastStartTime = span.end();
|
|
970
1001
|
return span.id;
|
|
971
1002
|
}
|
|
972
1003
|
/**
|
|
973
|
-
* Create a new toplevel span. The name
|
|
1004
|
+
* Create a new toplevel span underneath the experiment. The name defaults to "root".
|
|
974
1005
|
*
|
|
975
|
-
* See `Span.
|
|
976
|
-
*/
|
|
977
|
-
startSpan(args) {
|
|
978
|
-
this.checkNotFinished();
|
|
979
|
-
const { name, ...argsRest } = args ?? {};
|
|
980
|
-
return new SpanImpl({
|
|
981
|
-
bgLogger: this.bgLogger,
|
|
982
|
-
name: name ?? "root",
|
|
983
|
-
...argsRest,
|
|
984
|
-
rootExperiment: this
|
|
985
|
-
});
|
|
986
|
-
}
|
|
987
|
-
/**
|
|
988
|
-
* Wrapper over `Experiment.startSpan`, which passes the initialized `Span` it to the given callback and ends it afterwards. See `Span.traced` for full details.
|
|
1006
|
+
* See `Span.traced` for full details.
|
|
989
1007
|
*/
|
|
990
1008
|
traced(callback, args) {
|
|
991
1009
|
const { setCurrent, ...argsRest } = args ?? {};
|
|
@@ -993,7 +1011,7 @@ var Experiment = class {
|
|
|
993
1011
|
return runFinally(
|
|
994
1012
|
() => {
|
|
995
1013
|
if (setCurrent ?? true) {
|
|
996
|
-
return withCurrent(span,
|
|
1014
|
+
return withCurrent(span, callback);
|
|
997
1015
|
} else {
|
|
998
1016
|
return callback(span);
|
|
999
1017
|
}
|
|
@@ -1001,6 +1019,25 @@ var Experiment = class {
|
|
|
1001
1019
|
() => span.end()
|
|
1002
1020
|
);
|
|
1003
1021
|
}
|
|
1022
|
+
/**
|
|
1023
|
+
* Lower-level alternative to `traced`, which does not automatically end the span or mark it as current.
|
|
1024
|
+
*
|
|
1025
|
+
* See `traced` for full details.
|
|
1026
|
+
*/
|
|
1027
|
+
startSpan(args) {
|
|
1028
|
+
const { name, ...argsRest } = args ?? {};
|
|
1029
|
+
const parentIds = (async () => ({
|
|
1030
|
+
kind: "experiment",
|
|
1031
|
+
project_id: (await this.project).id,
|
|
1032
|
+
experiment_id: await this.id
|
|
1033
|
+
}))();
|
|
1034
|
+
return new SpanImpl({
|
|
1035
|
+
parentIds,
|
|
1036
|
+
bgLogger: this.bgLogger,
|
|
1037
|
+
name: name ?? "root",
|
|
1038
|
+
...argsRest
|
|
1039
|
+
});
|
|
1040
|
+
}
|
|
1004
1041
|
/**
|
|
1005
1042
|
* Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
1006
1043
|
*
|
|
@@ -1012,18 +1049,21 @@ var Experiment = class {
|
|
|
1012
1049
|
async summarize(options = {}) {
|
|
1013
1050
|
let { summarizeScores = true, comparisonExperimentId = void 0 } = options || {};
|
|
1014
1051
|
await this.bgLogger.flush();
|
|
1015
|
-
const
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1052
|
+
const state = await this.getState();
|
|
1053
|
+
const projectUrl = `${state.apiUrl}/app/${encodeURIComponent(
|
|
1054
|
+
state.orgName
|
|
1055
|
+
)}/p/${encodeURIComponent((await this.project).name)}`;
|
|
1056
|
+
const experimentUrl = `${projectUrl}/${encodeURIComponent(
|
|
1057
|
+
await this.name
|
|
1058
|
+
)}`;
|
|
1019
1059
|
let scores = void 0;
|
|
1020
1060
|
let metrics = void 0;
|
|
1021
1061
|
let comparisonExperimentName = void 0;
|
|
1022
1062
|
if (summarizeScores) {
|
|
1023
1063
|
if (comparisonExperimentId === void 0) {
|
|
1024
|
-
const conn =
|
|
1064
|
+
const conn = state.logConn();
|
|
1025
1065
|
const resp = await conn.get("/crud/base_experiments", {
|
|
1026
|
-
id: this.id
|
|
1066
|
+
id: await this.id
|
|
1027
1067
|
});
|
|
1028
1068
|
const base_experiments = await resp.json();
|
|
1029
1069
|
if (base_experiments.length > 0) {
|
|
@@ -1032,10 +1072,10 @@ var Experiment = class {
|
|
|
1032
1072
|
}
|
|
1033
1073
|
}
|
|
1034
1074
|
if (comparisonExperimentId !== void 0) {
|
|
1035
|
-
const results = await
|
|
1075
|
+
const results = await state.logConn().get_json(
|
|
1036
1076
|
"/experiment-comparison2",
|
|
1037
1077
|
{
|
|
1038
|
-
experiment_id: this.id,
|
|
1078
|
+
experiment_id: await this.id,
|
|
1039
1079
|
base_experiment_id: comparisonExperimentId
|
|
1040
1080
|
},
|
|
1041
1081
|
3
|
|
@@ -1045,8 +1085,8 @@ var Experiment = class {
|
|
|
1045
1085
|
}
|
|
1046
1086
|
}
|
|
1047
1087
|
return {
|
|
1048
|
-
projectName: this.project.name,
|
|
1049
|
-
experimentName: this.name,
|
|
1088
|
+
projectName: (await this.project).name,
|
|
1089
|
+
experimentName: await this.name,
|
|
1050
1090
|
projectUrl,
|
|
1051
1091
|
experimentUrl,
|
|
1052
1092
|
comparisonExperimentName,
|
|
@@ -1055,118 +1095,99 @@ var Experiment = class {
|
|
|
1055
1095
|
};
|
|
1056
1096
|
}
|
|
1057
1097
|
/**
|
|
1058
|
-
*
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1098
|
+
* Flush any pending rows to the server.
|
|
1099
|
+
*/
|
|
1100
|
+
async flush() {
|
|
1101
|
+
return await this.bgLogger.flush();
|
|
1102
|
+
}
|
|
1103
|
+
/**
|
|
1104
|
+
* This function is deprecated. You can simply remove it from your code.
|
|
1063
1105
|
*/
|
|
1064
1106
|
async close() {
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
unterminatedObjects.removeUnterminated(this);
|
|
1107
|
+
console.warn(
|
|
1108
|
+
"close is deprecated and will be removed in a future version of braintrust. It is now a no-op and can be removed"
|
|
1109
|
+
);
|
|
1069
1110
|
return this.id;
|
|
1070
1111
|
}
|
|
1071
|
-
checkNotFinished() {
|
|
1072
|
-
if (this.finished) {
|
|
1073
|
-
throw new Error("Cannot invoke method on finished experiment");
|
|
1074
|
-
}
|
|
1075
|
-
}
|
|
1076
1112
|
};
|
|
1077
1113
|
var SpanImpl = class _SpanImpl {
|
|
1078
1114
|
// root_experiment should only be specified for a root span. parent_span
|
|
1079
1115
|
// should only be specified for non-root spans.
|
|
1080
1116
|
constructor(args) {
|
|
1081
1117
|
this.kind = "span";
|
|
1082
|
-
this.finished = false;
|
|
1083
1118
|
this.loggedEndTime = void 0;
|
|
1084
1119
|
this.bgLogger = args.bgLogger;
|
|
1085
1120
|
const callerLocation = isomorph_default.getCallerLocation();
|
|
1121
|
+
const name = (() => {
|
|
1122
|
+
if (args.name)
|
|
1123
|
+
return args.name;
|
|
1124
|
+
if (callerLocation) {
|
|
1125
|
+
const pathComponents = callerLocation.caller_filename.split("/");
|
|
1126
|
+
const filename = pathComponents[pathComponents.length - 1];
|
|
1127
|
+
return [callerLocation.caller_functionname].concat(
|
|
1128
|
+
filename ? [`${filename}:${callerLocation.caller_lineno}`] : []
|
|
1129
|
+
).join(":");
|
|
1130
|
+
}
|
|
1131
|
+
return "subspan";
|
|
1132
|
+
})();
|
|
1086
1133
|
this.internalData = {
|
|
1087
1134
|
metrics: {
|
|
1088
1135
|
start: args.startTime ?? getCurrentUnixTimestamp(),
|
|
1089
1136
|
...callerLocation
|
|
1090
1137
|
},
|
|
1091
|
-
span_attributes: { ...args.spanAttributes, name
|
|
1138
|
+
span_attributes: { ...args.spanAttributes, name },
|
|
1092
1139
|
created: (/* @__PURE__ */ new Date()).toISOString()
|
|
1093
1140
|
};
|
|
1141
|
+
this.parentIds = args.parentIds;
|
|
1094
1142
|
const id = args.event?.id ?? v4_default();
|
|
1095
1143
|
const span_id = v4_default();
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
};
|
|
1104
|
-
} else if ("rootProject" in args) {
|
|
1105
|
-
this._object_info = {
|
|
1106
|
-
id,
|
|
1107
|
-
span_id,
|
|
1108
|
-
root_span_id: span_id,
|
|
1109
|
-
org_id: _state.orgId,
|
|
1110
|
-
project_id: args.rootProject.id,
|
|
1111
|
-
log_id: "g"
|
|
1112
|
-
};
|
|
1113
|
-
} else if ("parentSpan" in args) {
|
|
1114
|
-
this._object_info = {
|
|
1115
|
-
...args.parentSpan._object_info,
|
|
1116
|
-
id,
|
|
1117
|
-
span_id
|
|
1118
|
-
};
|
|
1119
|
-
this.internalData.span_parents = [args.parentSpan.span_id];
|
|
1120
|
-
} else {
|
|
1121
|
-
throw new Error("Must provide either 'rootExperiment' or 'parentSpan'");
|
|
1144
|
+
this.rowIds = {
|
|
1145
|
+
id,
|
|
1146
|
+
span_id,
|
|
1147
|
+
root_span_id: args.parentSpanInfo?.root_span_id ?? span_id
|
|
1148
|
+
};
|
|
1149
|
+
if (args.parentSpanInfo) {
|
|
1150
|
+
this.internalData.span_parents = [args.parentSpanInfo.span_id];
|
|
1122
1151
|
}
|
|
1123
1152
|
this.isMerge = false;
|
|
1124
1153
|
const { id: _id, ...eventRest } = args.event ?? {};
|
|
1125
1154
|
this.log(eventRest);
|
|
1126
1155
|
this.isMerge = true;
|
|
1127
|
-
unterminatedObjects.addUnterminated(this, callerLocation);
|
|
1128
1156
|
}
|
|
1129
1157
|
get id() {
|
|
1130
|
-
return this.
|
|
1158
|
+
return this.rowIds.id;
|
|
1131
1159
|
}
|
|
1132
1160
|
get span_id() {
|
|
1133
|
-
return this.
|
|
1161
|
+
return this.rowIds.span_id;
|
|
1134
1162
|
}
|
|
1135
1163
|
get root_span_id() {
|
|
1136
|
-
return this.
|
|
1164
|
+
return this.rowIds.root_span_id;
|
|
1137
1165
|
}
|
|
1138
1166
|
log(event) {
|
|
1139
|
-
this.checkNotFinished();
|
|
1140
1167
|
const sanitized = validateAndSanitizeExperimentLogPartialArgs(event);
|
|
1141
1168
|
const sanitizedAndInternalData = { ...this.internalData };
|
|
1142
1169
|
mergeDicts(sanitizedAndInternalData, sanitized);
|
|
1143
|
-
const record = {
|
|
1144
|
-
...sanitizedAndInternalData,
|
|
1145
|
-
...this._object_info,
|
|
1146
|
-
[IS_MERGE_FIELD]: this.isMerge
|
|
1147
|
-
};
|
|
1148
|
-
if (record.metrics?.end) {
|
|
1149
|
-
this.loggedEndTime = record.metrics?.end;
|
|
1150
|
-
}
|
|
1151
1170
|
this.internalData = {};
|
|
1171
|
+
if (sanitizedAndInternalData.metrics?.end) {
|
|
1172
|
+
this.loggedEndTime = sanitizedAndInternalData.metrics?.end;
|
|
1173
|
+
}
|
|
1174
|
+
const record = (async () => {
|
|
1175
|
+
return {
|
|
1176
|
+
...sanitizedAndInternalData,
|
|
1177
|
+
...this.rowIds,
|
|
1178
|
+
...await this.parentIds,
|
|
1179
|
+
[IS_MERGE_FIELD]: this.isMerge
|
|
1180
|
+
};
|
|
1181
|
+
})();
|
|
1152
1182
|
this.bgLogger.log([record]);
|
|
1153
1183
|
}
|
|
1154
|
-
|
|
1155
|
-
this.checkNotFinished();
|
|
1156
|
-
return new _SpanImpl({
|
|
1157
|
-
bgLogger: this.bgLogger,
|
|
1158
|
-
name,
|
|
1159
|
-
...args,
|
|
1160
|
-
parentSpan: this
|
|
1161
|
-
});
|
|
1162
|
-
}
|
|
1163
|
-
traced(name, callback, args) {
|
|
1184
|
+
traced(callback, args) {
|
|
1164
1185
|
const { setCurrent, ...argsRest } = args ?? {};
|
|
1165
|
-
const span = this.startSpan(
|
|
1186
|
+
const span = this.startSpan(argsRest);
|
|
1166
1187
|
return runFinally(
|
|
1167
1188
|
() => {
|
|
1168
1189
|
if (setCurrent ?? true) {
|
|
1169
|
-
return withCurrent(span,
|
|
1190
|
+
return withCurrent(span, callback);
|
|
1170
1191
|
} else {
|
|
1171
1192
|
return callback(span);
|
|
1172
1193
|
}
|
|
@@ -1174,8 +1195,18 @@ var SpanImpl = class _SpanImpl {
|
|
|
1174
1195
|
() => span.end()
|
|
1175
1196
|
);
|
|
1176
1197
|
}
|
|
1198
|
+
startSpan(args) {
|
|
1199
|
+
return new _SpanImpl({
|
|
1200
|
+
parentIds: this.parentIds,
|
|
1201
|
+
bgLogger: this.bgLogger,
|
|
1202
|
+
parentSpanInfo: {
|
|
1203
|
+
span_id: this.rowIds.span_id,
|
|
1204
|
+
root_span_id: this.rowIds.root_span_id
|
|
1205
|
+
},
|
|
1206
|
+
...args
|
|
1207
|
+
});
|
|
1208
|
+
}
|
|
1177
1209
|
end(args) {
|
|
1178
|
-
this.checkNotFinished();
|
|
1179
1210
|
let endTime;
|
|
1180
1211
|
if (!this.loggedEndTime) {
|
|
1181
1212
|
endTime = args?.endTime ?? getCurrentUnixTimestamp();
|
|
@@ -1184,45 +1215,38 @@ var SpanImpl = class _SpanImpl {
|
|
|
1184
1215
|
endTime = this.loggedEndTime;
|
|
1185
1216
|
}
|
|
1186
1217
|
this.log({});
|
|
1187
|
-
this.finished = true;
|
|
1188
|
-
unterminatedObjects.removeUnterminated(this);
|
|
1189
1218
|
return endTime;
|
|
1190
1219
|
}
|
|
1191
1220
|
close(args) {
|
|
1192
1221
|
return this.end(args);
|
|
1193
1222
|
}
|
|
1194
|
-
checkNotFinished() {
|
|
1195
|
-
if (this.finished) {
|
|
1196
|
-
throw new Error("Cannot invoke method on finished span");
|
|
1197
|
-
}
|
|
1198
|
-
}
|
|
1199
1223
|
};
|
|
1200
|
-
async function _initDataset(project_name, {
|
|
1201
|
-
name,
|
|
1202
|
-
description,
|
|
1203
|
-
version
|
|
1204
|
-
} = {}) {
|
|
1205
|
-
const args = {
|
|
1206
|
-
org_id: _state.orgId,
|
|
1207
|
-
project_name,
|
|
1208
|
-
dataset_name: name,
|
|
1209
|
-
description
|
|
1210
|
-
};
|
|
1211
|
-
const response = await _state.apiConn().post_json("api/dataset/register", args);
|
|
1212
|
-
const project = response.project;
|
|
1213
|
-
const dataset = response.dataset;
|
|
1214
|
-
return new Dataset(project, dataset.id, dataset.name, version);
|
|
1215
|
-
}
|
|
1216
1224
|
var Dataset = class {
|
|
1217
|
-
constructor(
|
|
1225
|
+
constructor(lazyMetadata, pinnedVersion) {
|
|
1218
1226
|
this._fetchedData = void 0;
|
|
1219
|
-
this.
|
|
1220
|
-
this.project = project;
|
|
1221
|
-
this.id = id;
|
|
1222
|
-
this.name = name;
|
|
1227
|
+
this.lazyMetadata = lazyMetadata;
|
|
1223
1228
|
this.pinnedVersion = pinnedVersion;
|
|
1224
|
-
this.
|
|
1225
|
-
|
|
1229
|
+
const logConn = this.getState().then((state) => state.logConn());
|
|
1230
|
+
this.bgLogger = new BackgroundLogger(logConn);
|
|
1231
|
+
}
|
|
1232
|
+
get id() {
|
|
1233
|
+
return (async () => {
|
|
1234
|
+
return (await this.lazyMetadata).dataset.id;
|
|
1235
|
+
})();
|
|
1236
|
+
}
|
|
1237
|
+
get name() {
|
|
1238
|
+
return (async () => {
|
|
1239
|
+
return (await this.lazyMetadata).dataset.name;
|
|
1240
|
+
})();
|
|
1241
|
+
}
|
|
1242
|
+
get project() {
|
|
1243
|
+
return (async () => {
|
|
1244
|
+
return (await this.lazyMetadata).project;
|
|
1245
|
+
})();
|
|
1246
|
+
}
|
|
1247
|
+
async getState() {
|
|
1248
|
+
await this.lazyMetadata;
|
|
1249
|
+
return _state;
|
|
1226
1250
|
}
|
|
1227
1251
|
/**
|
|
1228
1252
|
* Insert a single record to the dataset. The record will be batched and uploaded behind the scenes. If you pass in an `id`,
|
|
@@ -1244,7 +1268,6 @@ var Dataset = class {
|
|
|
1244
1268
|
metadata,
|
|
1245
1269
|
id
|
|
1246
1270
|
}) {
|
|
1247
|
-
this.checkNotFinished();
|
|
1248
1271
|
if (metadata !== void 0) {
|
|
1249
1272
|
for (const key of Object.keys(metadata)) {
|
|
1250
1273
|
if (typeof key !== "string") {
|
|
@@ -1252,29 +1275,29 @@ var Dataset = class {
|
|
|
1252
1275
|
}
|
|
1253
1276
|
}
|
|
1254
1277
|
}
|
|
1255
|
-
const
|
|
1256
|
-
|
|
1278
|
+
const rowId = id || v4_default();
|
|
1279
|
+
const args = (async () => ({
|
|
1280
|
+
id: rowId,
|
|
1257
1281
|
inputs: input,
|
|
1258
1282
|
output,
|
|
1259
|
-
project_id: this.project.id,
|
|
1260
|
-
dataset_id: this.id,
|
|
1283
|
+
project_id: (await this.project).id,
|
|
1284
|
+
dataset_id: await this.id,
|
|
1261
1285
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1262
1286
|
metadata
|
|
1263
|
-
};
|
|
1264
|
-
this.
|
|
1265
|
-
return
|
|
1287
|
+
}))();
|
|
1288
|
+
this.bgLogger.log([args]);
|
|
1289
|
+
return rowId;
|
|
1266
1290
|
}
|
|
1267
1291
|
delete(id) {
|
|
1268
|
-
|
|
1269
|
-
const args = {
|
|
1292
|
+
const args = (async () => ({
|
|
1270
1293
|
id,
|
|
1271
|
-
project_id: this.project.id,
|
|
1272
|
-
dataset_id: this.id,
|
|
1294
|
+
project_id: (await this.project).id,
|
|
1295
|
+
dataset_id: await this.id,
|
|
1273
1296
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1274
1297
|
_object_delete: true
|
|
1275
|
-
};
|
|
1276
|
-
this.
|
|
1277
|
-
return
|
|
1298
|
+
}))();
|
|
1299
|
+
this.bgLogger.log([args]);
|
|
1300
|
+
return id;
|
|
1278
1301
|
}
|
|
1279
1302
|
/**
|
|
1280
1303
|
* Summarize the dataset, including high level metrics about its size and other metadata.
|
|
@@ -1283,26 +1306,26 @@ var Dataset = class {
|
|
|
1283
1306
|
* @returns A summary of the dataset.
|
|
1284
1307
|
*/
|
|
1285
1308
|
async summarize(options = {}) {
|
|
1286
|
-
this.checkNotFinished();
|
|
1287
1309
|
let { summarizeData = true } = options || {};
|
|
1288
|
-
await this.
|
|
1289
|
-
const
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1310
|
+
await this.bgLogger.flush();
|
|
1311
|
+
const state = await this.getState();
|
|
1312
|
+
const projectUrl = `${state.apiUrl}/app/${encodeURIComponent(
|
|
1313
|
+
state.orgName
|
|
1314
|
+
)}/p/${encodeURIComponent((await this.project).name)}`;
|
|
1315
|
+
const datasetUrl = `${projectUrl}/d/${encodeURIComponent(await this.name)}`;
|
|
1293
1316
|
let dataSummary = void 0;
|
|
1294
1317
|
if (summarizeData) {
|
|
1295
|
-
dataSummary = await
|
|
1318
|
+
dataSummary = await state.logConn().get_json(
|
|
1296
1319
|
"dataset-summary",
|
|
1297
1320
|
{
|
|
1298
|
-
dataset_id: this.id
|
|
1321
|
+
dataset_id: await this.id
|
|
1299
1322
|
},
|
|
1300
1323
|
3
|
|
1301
1324
|
);
|
|
1302
1325
|
}
|
|
1303
1326
|
return {
|
|
1304
|
-
projectName: this.project.name,
|
|
1305
|
-
datasetName: this.name,
|
|
1327
|
+
projectName: (await this.project).name,
|
|
1328
|
+
datasetName: await this.name,
|
|
1306
1329
|
projectUrl,
|
|
1307
1330
|
datasetUrl,
|
|
1308
1331
|
dataSummary
|
|
@@ -1327,7 +1350,6 @@ var Dataset = class {
|
|
|
1327
1350
|
* @returns An iterator over the dataset's records.
|
|
1328
1351
|
*/
|
|
1329
1352
|
async *fetch() {
|
|
1330
|
-
this.checkNotFinished();
|
|
1331
1353
|
const records = await this.fetchedData();
|
|
1332
1354
|
for (const record of records) {
|
|
1333
1355
|
yield {
|
|
@@ -1351,14 +1373,13 @@ var Dataset = class {
|
|
|
1351
1373
|
* ```
|
|
1352
1374
|
*/
|
|
1353
1375
|
[Symbol.asyncIterator]() {
|
|
1354
|
-
this.checkNotFinished();
|
|
1355
1376
|
return this.fetch();
|
|
1356
1377
|
}
|
|
1357
1378
|
async fetchedData() {
|
|
1358
|
-
this.checkNotFinished();
|
|
1359
1379
|
if (this._fetchedData === void 0) {
|
|
1360
|
-
const
|
|
1361
|
-
|
|
1380
|
+
const state = await this.getState();
|
|
1381
|
+
const resp = await state.logConn().get("object/dataset", {
|
|
1382
|
+
id: await this.id,
|
|
1362
1383
|
fmt: "json",
|
|
1363
1384
|
version: this.pinnedVersion
|
|
1364
1385
|
});
|
|
@@ -1368,11 +1389,9 @@ var Dataset = class {
|
|
|
1368
1389
|
return this._fetchedData || [];
|
|
1369
1390
|
}
|
|
1370
1391
|
clearCache() {
|
|
1371
|
-
this.checkNotFinished();
|
|
1372
1392
|
this._fetchedData = void 0;
|
|
1373
1393
|
}
|
|
1374
1394
|
async version() {
|
|
1375
|
-
this.checkNotFinished();
|
|
1376
1395
|
if (this.pinnedVersion !== void 0) {
|
|
1377
1396
|
return this.pinnedVersion;
|
|
1378
1397
|
} else {
|
|
@@ -1388,24 +1407,20 @@ var Dataset = class {
|
|
|
1388
1407
|
}
|
|
1389
1408
|
}
|
|
1390
1409
|
/**
|
|
1391
|
-
*
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1410
|
+
* Flush any pending rows to the server.
|
|
1411
|
+
*/
|
|
1412
|
+
async flush() {
|
|
1413
|
+
return await this.bgLogger.flush();
|
|
1414
|
+
}
|
|
1415
|
+
/**
|
|
1416
|
+
* This function is deprecated. You can simply remove it from your code.
|
|
1396
1417
|
*/
|
|
1397
1418
|
async close() {
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
unterminatedObjects.removeUnterminated(this);
|
|
1419
|
+
console.warn(
|
|
1420
|
+
"close is deprecated and will be removed in a future version of braintrust. It is now a no-op and can be removed"
|
|
1421
|
+
);
|
|
1402
1422
|
return this.id;
|
|
1403
1423
|
}
|
|
1404
|
-
checkNotFinished() {
|
|
1405
|
-
if (this.finished) {
|
|
1406
|
-
throw new Error("Cannot invoke method on finished dataset");
|
|
1407
|
-
}
|
|
1408
|
-
}
|
|
1409
1424
|
};
|
|
1410
1425
|
|
|
1411
1426
|
// src/browser-config.ts
|
|
@@ -1606,6 +1621,7 @@ export {
|
|
|
1606
1621
|
Dataset,
|
|
1607
1622
|
Experiment,
|
|
1608
1623
|
Logger,
|
|
1624
|
+
NOOP_SPAN,
|
|
1609
1625
|
NoopSpan,
|
|
1610
1626
|
SpanImpl,
|
|
1611
1627
|
_internalGetGlobalState,
|
|
@@ -1613,16 +1629,15 @@ export {
|
|
|
1613
1629
|
currentExperiment,
|
|
1614
1630
|
currentLogger,
|
|
1615
1631
|
currentSpan,
|
|
1632
|
+
getSpanParentObject,
|
|
1616
1633
|
init,
|
|
1617
1634
|
initDataset,
|
|
1618
1635
|
initLogger,
|
|
1619
1636
|
log,
|
|
1620
1637
|
login,
|
|
1621
|
-
noopSpan,
|
|
1622
1638
|
startSpan,
|
|
1623
1639
|
summarize,
|
|
1624
1640
|
traced,
|
|
1625
|
-
withCurrent,
|
|
1626
1641
|
withDataset,
|
|
1627
1642
|
withExperiment,
|
|
1628
1643
|
withLogger,
|