braintrust 3.12.0 → 3.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dev/dist/index.d.mts +25 -6
- package/dev/dist/index.d.ts +25 -6
- package/dev/dist/index.js +134 -43
- package/dev/dist/index.mjs +113 -22
- package/dist/apply-auto-instrumentation.js +170 -170
- package/dist/apply-auto-instrumentation.mjs +1 -1
- package/dist/auto-instrumentations/bundler/esbuild.cjs +1 -0
- package/dist/auto-instrumentations/bundler/esbuild.mjs +2 -2
- package/dist/auto-instrumentations/bundler/next.cjs +1 -0
- package/dist/auto-instrumentations/bundler/next.mjs +3 -3
- package/dist/auto-instrumentations/bundler/rollup.cjs +1 -0
- package/dist/auto-instrumentations/bundler/rollup.mjs +2 -2
- package/dist/auto-instrumentations/bundler/vite.cjs +1 -0
- package/dist/auto-instrumentations/bundler/vite.mjs +2 -2
- package/dist/auto-instrumentations/bundler/webpack-loader.cjs +1 -0
- package/dist/auto-instrumentations/bundler/webpack.cjs +1 -0
- package/dist/auto-instrumentations/bundler/webpack.mjs +3 -3
- package/dist/auto-instrumentations/{chunk-2DPA74KK.mjs → chunk-E5DUYJWK.mjs} +1 -0
- package/dist/auto-instrumentations/{chunk-73BZUKVI.mjs → chunk-GJOO4ESL.mjs} +1 -1
- package/dist/auto-instrumentations/{chunk-AFXRW7I7.mjs → chunk-WFEUJACP.mjs} +1 -1
- package/dist/auto-instrumentations/hook.mjs +1 -0
- package/dist/auto-instrumentations/index.cjs +1 -0
- package/dist/auto-instrumentations/index.mjs +1 -1
- package/dist/browser.d.mts +149 -21
- package/dist/browser.d.ts +149 -21
- package/dist/browser.js +114 -24
- package/dist/browser.mjs +114 -24
- package/dist/{chunk-BW4DF4CY.js → chunk-26JGOELH.js} +1 -0
- package/dist/{chunk-MSLBGITU.mjs → chunk-75IQCUB2.mjs} +1 -0
- package/dist/cli.js +121 -44
- package/dist/edge-light.d.mts +1 -1
- package/dist/edge-light.d.ts +1 -1
- package/dist/edge-light.js +114 -24
- package/dist/edge-light.mjs +114 -24
- package/dist/index.d.mts +149 -21
- package/dist/index.d.ts +149 -21
- package/dist/index.js +529 -394
- package/dist/index.mjs +161 -26
- package/dist/instrumentation/index.d.mts +40 -3
- package/dist/instrumentation/index.d.ts +40 -3
- package/dist/instrumentation/index.js +15 -2
- package/dist/instrumentation/index.mjs +15 -2
- package/dist/workerd.d.mts +1 -1
- package/dist/workerd.d.ts +1 -1
- package/dist/workerd.js +114 -24
- package/dist/workerd.mjs +114 -24
- package/package.json +3 -17
- package/util/dist/index.d.mts +3 -1
- package/util/dist/index.d.ts +3 -1
package/dist/browser.js
CHANGED
|
@@ -47,6 +47,7 @@ __export(browser_exports, {
|
|
|
47
47
|
DEFAULT_FETCH_BATCH_SIZE: () => DEFAULT_FETCH_BATCH_SIZE,
|
|
48
48
|
DEFAULT_MAX_REQUEST_SIZE: () => DEFAULT_MAX_REQUEST_SIZE,
|
|
49
49
|
Dataset: () => Dataset2,
|
|
50
|
+
DatasetPipeline: () => DatasetPipeline,
|
|
50
51
|
ERR_PERMALINK: () => ERR_PERMALINK,
|
|
51
52
|
Eval: () => Eval,
|
|
52
53
|
EvalResultWithSummary: () => EvalResultWithSummary,
|
|
@@ -270,6 +271,7 @@ var iso = {
|
|
|
270
271
|
getRepoInfo: async (_settings) => void 0,
|
|
271
272
|
getPastNAncestors: async () => [],
|
|
272
273
|
getEnv: (_name) => void 0,
|
|
274
|
+
getBraintrustApiKey: async () => void 0,
|
|
273
275
|
getCallerLocation: () => void 0,
|
|
274
276
|
newAsyncLocalStorage: () => new DefaultAsyncLocalStorage(),
|
|
275
277
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -5609,6 +5611,19 @@ var JSONAttachment = class extends Attachment {
|
|
|
5609
5611
|
*/
|
|
5610
5612
|
constructor(data, options) {
|
|
5611
5613
|
const { filename = "data.json", pretty = false, state } = options ?? {};
|
|
5614
|
+
const deferredJsonAttachment = globalThis.__BT_DATASET_PIPELINE_DEFER_JSON_ATTACHMENT__;
|
|
5615
|
+
if (deferredJsonAttachment) {
|
|
5616
|
+
super({
|
|
5617
|
+
data: new Blob([]),
|
|
5618
|
+
filename,
|
|
5619
|
+
contentType: "application/json",
|
|
5620
|
+
state
|
|
5621
|
+
});
|
|
5622
|
+
return deferredJsonAttachment(data, {
|
|
5623
|
+
filename,
|
|
5624
|
+
pretty
|
|
5625
|
+
});
|
|
5626
|
+
}
|
|
5612
5627
|
const jsonString = pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data);
|
|
5613
5628
|
const blob = new Blob([jsonString], { type: "application/json" });
|
|
5614
5629
|
super({
|
|
@@ -7601,10 +7616,11 @@ async function login(options = {}) {
|
|
|
7601
7616
|
async function loginToState(options = {}) {
|
|
7602
7617
|
const {
|
|
7603
7618
|
appUrl = isomorph_default.getEnv("BRAINTRUST_APP_URL") || "https://www.braintrust.dev",
|
|
7604
|
-
apiKey
|
|
7619
|
+
apiKey: apiKeyArg,
|
|
7605
7620
|
orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME"),
|
|
7606
7621
|
fetch: fetch2 = globalThis.fetch
|
|
7607
7622
|
} = options || {};
|
|
7623
|
+
const apiKey = apiKeyArg !== void 0 ? apiKeyArg : await isomorph_default.getBraintrustApiKey();
|
|
7608
7624
|
const appPublicUrl = isomorph_default.getEnv("BRAINTRUST_APP_PUBLIC_URL") || appUrl;
|
|
7609
7625
|
const state = new BraintrustState(options);
|
|
7610
7626
|
state.resetLoginInfo();
|
|
@@ -8845,9 +8861,15 @@ var SpanImpl = class _SpanImpl {
|
|
|
8845
8861
|
const cachedSpan = {
|
|
8846
8862
|
input: partialRecord.input,
|
|
8847
8863
|
output: partialRecord.output,
|
|
8864
|
+
expected: partialRecord.expected,
|
|
8865
|
+
error: partialRecord.error,
|
|
8866
|
+
scores: partialRecord.scores,
|
|
8867
|
+
metrics: partialRecord.metrics,
|
|
8848
8868
|
metadata: partialRecord.metadata,
|
|
8869
|
+
tags: partialRecord.tags,
|
|
8849
8870
|
span_id: this._spanId,
|
|
8850
8871
|
span_parents: this._spanParents,
|
|
8872
|
+
is_root: this._spanId === this._rootSpanId,
|
|
8851
8873
|
span_attributes: partialRecord.span_attributes
|
|
8852
8874
|
};
|
|
8853
8875
|
this._state.spanCache.queueWrite(
|
|
@@ -9183,6 +9205,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9183
9205
|
metadata,
|
|
9184
9206
|
tags,
|
|
9185
9207
|
output,
|
|
9208
|
+
origin,
|
|
9186
9209
|
isMerge
|
|
9187
9210
|
}) {
|
|
9188
9211
|
return new LazyValue(async () => {
|
|
@@ -9197,6 +9220,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9197
9220
|
created: !isMerge ? (/* @__PURE__ */ new Date()).toISOString() : void 0,
|
|
9198
9221
|
//if we're merging/updating an event we will not add this ts
|
|
9199
9222
|
metadata,
|
|
9223
|
+
origin,
|
|
9200
9224
|
...!!isMerge ? {
|
|
9201
9225
|
[IS_MERGE_FIELD]: true
|
|
9202
9226
|
} : {}
|
|
@@ -9216,6 +9240,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9216
9240
|
* about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
|
|
9217
9241
|
* `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
|
|
9218
9242
|
* JSON-serializable type, but its keys must be strings.
|
|
9243
|
+
* @param event.origin (Optional) a reference to the source object this dataset record was derived from.
|
|
9219
9244
|
* @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
|
|
9220
9245
|
* @param event.output: (Deprecated) The output of your application. Use `expected` instead.
|
|
9221
9246
|
* @returns The `id` of the logged record.
|
|
@@ -9226,7 +9251,8 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9226
9251
|
metadata,
|
|
9227
9252
|
tags,
|
|
9228
9253
|
id,
|
|
9229
|
-
output
|
|
9254
|
+
output,
|
|
9255
|
+
origin
|
|
9230
9256
|
}) {
|
|
9231
9257
|
this.validateEvent({ metadata, expected, output, tags });
|
|
9232
9258
|
const rowId = id || (0, import_uuid2.v4)();
|
|
@@ -9238,6 +9264,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9238
9264
|
metadata,
|
|
9239
9265
|
tags,
|
|
9240
9266
|
output,
|
|
9267
|
+
origin,
|
|
9241
9268
|
isMerge: false
|
|
9242
9269
|
})
|
|
9243
9270
|
);
|
|
@@ -26195,6 +26222,10 @@ function configureBrowser() {
|
|
|
26195
26222
|
}
|
|
26196
26223
|
return process.env[name];
|
|
26197
26224
|
};
|
|
26225
|
+
isomorph_default.getBraintrustApiKey = async () => {
|
|
26226
|
+
const value = isomorph_default.getEnv("BRAINTRUST_API_KEY");
|
|
26227
|
+
return value?.trim() ? value : void 0;
|
|
26228
|
+
};
|
|
26198
26229
|
isomorph_default.hash = (data) => {
|
|
26199
26230
|
let hash = 0;
|
|
26200
26231
|
for (let i = 0; i < data.length; i++) {
|
|
@@ -26230,6 +26261,7 @@ __export(exports_exports, {
|
|
|
26230
26261
|
DEFAULT_FETCH_BATCH_SIZE: () => DEFAULT_FETCH_BATCH_SIZE,
|
|
26231
26262
|
DEFAULT_MAX_REQUEST_SIZE: () => DEFAULT_MAX_REQUEST_SIZE,
|
|
26232
26263
|
Dataset: () => Dataset2,
|
|
26264
|
+
DatasetPipeline: () => DatasetPipeline,
|
|
26233
26265
|
ERR_PERMALINK: () => ERR_PERMALINK,
|
|
26234
26266
|
Eval: () => Eval,
|
|
26235
26267
|
EvalResultWithSummary: () => EvalResultWithSummary,
|
|
@@ -29488,10 +29520,12 @@ function formatExperimentSummary(summary) {
|
|
|
29488
29520
|
// src/wrappers/shared/flush.ts
|
|
29489
29521
|
async function summarizeAndFlush(experiment, options) {
|
|
29490
29522
|
const shouldDisplay = options.displaySummary ?? true;
|
|
29491
|
-
|
|
29492
|
-
|
|
29493
|
-
|
|
29523
|
+
if (!shouldDisplay) {
|
|
29524
|
+
await experiment.flush();
|
|
29525
|
+
return;
|
|
29494
29526
|
}
|
|
29527
|
+
const summary = await experiment.summarize();
|
|
29528
|
+
console.log(formatExperimentSummary(summary));
|
|
29495
29529
|
}
|
|
29496
29530
|
|
|
29497
29531
|
// src/wrappers/vitest/flush-manager.ts
|
|
@@ -31355,8 +31389,12 @@ var waterfall$1 = awaitify(waterfall);
|
|
|
31355
31389
|
|
|
31356
31390
|
// src/trace.ts
|
|
31357
31391
|
var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
31358
|
-
constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter) {
|
|
31359
|
-
const filterExpr = _SpanFetcher.buildFilter(
|
|
31392
|
+
constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter, includeScorers = false) {
|
|
31393
|
+
const filterExpr = _SpanFetcher.buildFilter(
|
|
31394
|
+
rootSpanId,
|
|
31395
|
+
spanTypeFilter,
|
|
31396
|
+
includeScorers
|
|
31397
|
+
);
|
|
31360
31398
|
super(objectType, void 0, void 0, {
|
|
31361
31399
|
filter: filterExpr
|
|
31362
31400
|
});
|
|
@@ -31365,16 +31403,17 @@ var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
|
31365
31403
|
this._state = _state;
|
|
31366
31404
|
this.spanTypeFilter = spanTypeFilter;
|
|
31367
31405
|
}
|
|
31368
|
-
static buildFilter(rootSpanId, spanTypeFilter) {
|
|
31406
|
+
static buildFilter(rootSpanId, spanTypeFilter, includeScorers = false) {
|
|
31369
31407
|
const children = [
|
|
31370
31408
|
// Base filter: root_span_id = 'value'
|
|
31371
31409
|
{
|
|
31372
31410
|
op: "eq",
|
|
31373
31411
|
left: { op: "ident", name: ["root_span_id"] },
|
|
31374
31412
|
right: { op: "literal", value: rootSpanId }
|
|
31375
|
-
}
|
|
31376
|
-
|
|
31377
|
-
|
|
31413
|
+
}
|
|
31414
|
+
];
|
|
31415
|
+
if (!includeScorers) {
|
|
31416
|
+
children.push({
|
|
31378
31417
|
op: "or",
|
|
31379
31418
|
children: [
|
|
31380
31419
|
{
|
|
@@ -31387,8 +31426,8 @@ var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
|
31387
31426
|
right: { op: "literal", value: "scorer" }
|
|
31388
31427
|
}
|
|
31389
31428
|
]
|
|
31390
|
-
}
|
|
31391
|
-
|
|
31429
|
+
});
|
|
31430
|
+
}
|
|
31392
31431
|
if (spanTypeFilter && spanTypeFilter.length > 0) {
|
|
31393
31432
|
children.push({
|
|
31394
31433
|
op: "in",
|
|
@@ -31414,35 +31453,49 @@ var CachedSpanFetcher = class {
|
|
|
31414
31453
|
fetchFn;
|
|
31415
31454
|
constructor(objectTypeOrFetchFn, objectId, rootSpanId, getState) {
|
|
31416
31455
|
if (typeof objectTypeOrFetchFn === "function") {
|
|
31417
|
-
this.fetchFn = objectTypeOrFetchFn;
|
|
31456
|
+
this.fetchFn = (spanType) => objectTypeOrFetchFn(spanType);
|
|
31418
31457
|
} else {
|
|
31419
31458
|
const objectType = objectTypeOrFetchFn;
|
|
31420
|
-
this.fetchFn = async (spanType) => {
|
|
31459
|
+
this.fetchFn = async (spanType, includeScorers) => {
|
|
31421
31460
|
const state = await getState();
|
|
31422
31461
|
const fetcher = new SpanFetcher(
|
|
31423
31462
|
objectType,
|
|
31424
31463
|
objectId,
|
|
31425
31464
|
rootSpanId,
|
|
31426
31465
|
state,
|
|
31427
|
-
spanType
|
|
31466
|
+
spanType,
|
|
31467
|
+
includeScorers
|
|
31428
31468
|
);
|
|
31429
31469
|
const rows = await fetcher.fetchedData();
|
|
31430
|
-
return rows.
|
|
31470
|
+
return rows.map((row) => ({
|
|
31431
31471
|
input: row.input,
|
|
31432
31472
|
output: row.output,
|
|
31473
|
+
expected: row.expected,
|
|
31474
|
+
error: row.error,
|
|
31475
|
+
scores: row.scores,
|
|
31476
|
+
metrics: row.metrics,
|
|
31433
31477
|
metadata: row.metadata,
|
|
31434
31478
|
span_id: row.span_id,
|
|
31435
31479
|
span_parents: row.span_parents,
|
|
31480
|
+
is_root: row.is_root,
|
|
31436
31481
|
span_attributes: row.span_attributes,
|
|
31437
31482
|
id: row.id,
|
|
31438
31483
|
_xact_id: row._xact_id,
|
|
31439
31484
|
_pagination_key: row._pagination_key,
|
|
31440
|
-
root_span_id: row.root_span_id
|
|
31485
|
+
root_span_id: row.root_span_id,
|
|
31486
|
+
created: row.created,
|
|
31487
|
+
tags: row.tags
|
|
31441
31488
|
}));
|
|
31442
31489
|
};
|
|
31443
31490
|
}
|
|
31444
31491
|
}
|
|
31445
|
-
async getSpans({
|
|
31492
|
+
async getSpans({
|
|
31493
|
+
spanType,
|
|
31494
|
+
includeScorers = false
|
|
31495
|
+
} = {}) {
|
|
31496
|
+
if (includeScorers) {
|
|
31497
|
+
return this.fetchFn(spanType, true);
|
|
31498
|
+
}
|
|
31446
31499
|
if (this.allFetched) {
|
|
31447
31500
|
return this.getFromCache(spanType);
|
|
31448
31501
|
}
|
|
@@ -31459,7 +31512,7 @@ var CachedSpanFetcher = class {
|
|
|
31459
31512
|
return this.getFromCache(spanType);
|
|
31460
31513
|
}
|
|
31461
31514
|
async fetchSpans(spanType) {
|
|
31462
|
-
const spans = await this.fetchFn(spanType);
|
|
31515
|
+
const spans = await this.fetchFn(spanType, false);
|
|
31463
31516
|
for (const span of spans) {
|
|
31464
31517
|
const type = span.span_attributes?.type ?? "";
|
|
31465
31518
|
const existing = this.spanCache.get(type) ?? [];
|
|
@@ -31537,10 +31590,13 @@ var LocalTrace = class {
|
|
|
31537
31590
|
* First checks the local span cache for recently logged spans, then falls
|
|
31538
31591
|
* back to CachedSpanFetcher which handles BTQL fetching and caching.
|
|
31539
31592
|
*/
|
|
31540
|
-
async getSpans({
|
|
31593
|
+
async getSpans({
|
|
31594
|
+
spanType,
|
|
31595
|
+
includeScorers = false
|
|
31596
|
+
} = {}) {
|
|
31541
31597
|
const cachedSpans = this.state.spanCache.getByRootSpanId(this.rootSpanId);
|
|
31542
31598
|
if (cachedSpans && cachedSpans.length > 0) {
|
|
31543
|
-
let spans = cachedSpans.filter(
|
|
31599
|
+
let spans = includeScorers ? cachedSpans : cachedSpans.filter(
|
|
31544
31600
|
(span) => span.span_attributes?.purpose !== "scorer"
|
|
31545
31601
|
);
|
|
31546
31602
|
if (spanType && spanType.length > 0) {
|
|
@@ -31551,13 +31607,19 @@ var LocalTrace = class {
|
|
|
31551
31607
|
return spans.map((span) => ({
|
|
31552
31608
|
input: span.input,
|
|
31553
31609
|
output: span.output,
|
|
31610
|
+
expected: span.expected,
|
|
31611
|
+
error: span.error,
|
|
31612
|
+
scores: span.scores,
|
|
31613
|
+
metrics: span.metrics,
|
|
31554
31614
|
metadata: span.metadata,
|
|
31555
31615
|
span_id: span.span_id,
|
|
31556
31616
|
span_parents: span.span_parents,
|
|
31557
|
-
|
|
31617
|
+
is_root: span.is_root,
|
|
31618
|
+
span_attributes: span.span_attributes,
|
|
31619
|
+
tags: span.tags
|
|
31558
31620
|
}));
|
|
31559
31621
|
}
|
|
31560
|
-
return this.cachedFetcher.getSpans({ spanType });
|
|
31622
|
+
return this.cachedFetcher.getSpans({ spanType, includeScorers });
|
|
31561
31623
|
}
|
|
31562
31624
|
/**
|
|
31563
31625
|
* Get the thread (preprocessed messages) for this trace.
|
|
@@ -32746,6 +32808,34 @@ var defaultReporter = {
|
|
|
32746
32808
|
}
|
|
32747
32809
|
};
|
|
32748
32810
|
|
|
32811
|
+
// src/dataset-pipeline.ts
|
|
32812
|
+
function DatasetPipeline(definition) {
|
|
32813
|
+
if (!globalThis.__braintrust_dataset_pipelines) {
|
|
32814
|
+
globalThis.__braintrust_dataset_pipelines = [];
|
|
32815
|
+
}
|
|
32816
|
+
const storedDefinition = {
|
|
32817
|
+
name: definition.name,
|
|
32818
|
+
source: {
|
|
32819
|
+
projectId: definition.source.projectId,
|
|
32820
|
+
projectName: definition.source.projectName,
|
|
32821
|
+
orgName: definition.source.orgName,
|
|
32822
|
+
filter: definition.source.filter,
|
|
32823
|
+
scope: definition.source.scope ?? "span"
|
|
32824
|
+
},
|
|
32825
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions, @typescript-eslint/no-explicit-any
|
|
32826
|
+
transform: definition.transform,
|
|
32827
|
+
target: {
|
|
32828
|
+
projectId: definition.target.projectId,
|
|
32829
|
+
projectName: definition.target.projectName,
|
|
32830
|
+
orgName: definition.target.orgName,
|
|
32831
|
+
datasetName: definition.target.datasetName,
|
|
32832
|
+
description: definition.target.description,
|
|
32833
|
+
metadata: definition.target.metadata
|
|
32834
|
+
}
|
|
32835
|
+
};
|
|
32836
|
+
globalThis.__braintrust_dataset_pipelines.push(storedDefinition);
|
|
32837
|
+
}
|
|
32838
|
+
|
|
32749
32839
|
// src/framework2.ts
|
|
32750
32840
|
var import_v312 = require("zod/v3");
|
|
32751
32841
|
var currentFilename = typeof __filename !== "undefined" ? __filename : "unknown";
|
package/dist/browser.mjs
CHANGED
|
@@ -92,6 +92,7 @@ var iso = {
|
|
|
92
92
|
getRepoInfo: async (_settings) => void 0,
|
|
93
93
|
getPastNAncestors: async () => [],
|
|
94
94
|
getEnv: (_name) => void 0,
|
|
95
|
+
getBraintrustApiKey: async () => void 0,
|
|
95
96
|
getCallerLocation: () => void 0,
|
|
96
97
|
newAsyncLocalStorage: () => new DefaultAsyncLocalStorage(),
|
|
97
98
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -5433,6 +5434,19 @@ var JSONAttachment = class extends Attachment {
|
|
|
5433
5434
|
*/
|
|
5434
5435
|
constructor(data, options) {
|
|
5435
5436
|
const { filename = "data.json", pretty = false, state } = options ?? {};
|
|
5437
|
+
const deferredJsonAttachment = globalThis.__BT_DATASET_PIPELINE_DEFER_JSON_ATTACHMENT__;
|
|
5438
|
+
if (deferredJsonAttachment) {
|
|
5439
|
+
super({
|
|
5440
|
+
data: new Blob([]),
|
|
5441
|
+
filename,
|
|
5442
|
+
contentType: "application/json",
|
|
5443
|
+
state
|
|
5444
|
+
});
|
|
5445
|
+
return deferredJsonAttachment(data, {
|
|
5446
|
+
filename,
|
|
5447
|
+
pretty
|
|
5448
|
+
});
|
|
5449
|
+
}
|
|
5436
5450
|
const jsonString = pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data);
|
|
5437
5451
|
const blob = new Blob([jsonString], { type: "application/json" });
|
|
5438
5452
|
super({
|
|
@@ -7425,10 +7439,11 @@ async function login(options = {}) {
|
|
|
7425
7439
|
async function loginToState(options = {}) {
|
|
7426
7440
|
const {
|
|
7427
7441
|
appUrl = isomorph_default.getEnv("BRAINTRUST_APP_URL") || "https://www.braintrust.dev",
|
|
7428
|
-
apiKey
|
|
7442
|
+
apiKey: apiKeyArg,
|
|
7429
7443
|
orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME"),
|
|
7430
7444
|
fetch: fetch2 = globalThis.fetch
|
|
7431
7445
|
} = options || {};
|
|
7446
|
+
const apiKey = apiKeyArg !== void 0 ? apiKeyArg : await isomorph_default.getBraintrustApiKey();
|
|
7432
7447
|
const appPublicUrl = isomorph_default.getEnv("BRAINTRUST_APP_PUBLIC_URL") || appUrl;
|
|
7433
7448
|
const state = new BraintrustState(options);
|
|
7434
7449
|
state.resetLoginInfo();
|
|
@@ -8669,9 +8684,15 @@ var SpanImpl = class _SpanImpl {
|
|
|
8669
8684
|
const cachedSpan = {
|
|
8670
8685
|
input: partialRecord.input,
|
|
8671
8686
|
output: partialRecord.output,
|
|
8687
|
+
expected: partialRecord.expected,
|
|
8688
|
+
error: partialRecord.error,
|
|
8689
|
+
scores: partialRecord.scores,
|
|
8690
|
+
metrics: partialRecord.metrics,
|
|
8672
8691
|
metadata: partialRecord.metadata,
|
|
8692
|
+
tags: partialRecord.tags,
|
|
8673
8693
|
span_id: this._spanId,
|
|
8674
8694
|
span_parents: this._spanParents,
|
|
8695
|
+
is_root: this._spanId === this._rootSpanId,
|
|
8675
8696
|
span_attributes: partialRecord.span_attributes
|
|
8676
8697
|
};
|
|
8677
8698
|
this._state.spanCache.queueWrite(
|
|
@@ -9007,6 +9028,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9007
9028
|
metadata,
|
|
9008
9029
|
tags,
|
|
9009
9030
|
output,
|
|
9031
|
+
origin,
|
|
9010
9032
|
isMerge
|
|
9011
9033
|
}) {
|
|
9012
9034
|
return new LazyValue(async () => {
|
|
@@ -9021,6 +9043,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9021
9043
|
created: !isMerge ? (/* @__PURE__ */ new Date()).toISOString() : void 0,
|
|
9022
9044
|
//if we're merging/updating an event we will not add this ts
|
|
9023
9045
|
metadata,
|
|
9046
|
+
origin,
|
|
9024
9047
|
...!!isMerge ? {
|
|
9025
9048
|
[IS_MERGE_FIELD]: true
|
|
9026
9049
|
} : {}
|
|
@@ -9040,6 +9063,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9040
9063
|
* about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
|
|
9041
9064
|
* `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
|
|
9042
9065
|
* JSON-serializable type, but its keys must be strings.
|
|
9066
|
+
* @param event.origin (Optional) a reference to the source object this dataset record was derived from.
|
|
9043
9067
|
* @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
|
|
9044
9068
|
* @param event.output: (Deprecated) The output of your application. Use `expected` instead.
|
|
9045
9069
|
* @returns The `id` of the logged record.
|
|
@@ -9050,7 +9074,8 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9050
9074
|
metadata,
|
|
9051
9075
|
tags,
|
|
9052
9076
|
id,
|
|
9053
|
-
output
|
|
9077
|
+
output,
|
|
9078
|
+
origin
|
|
9054
9079
|
}) {
|
|
9055
9080
|
this.validateEvent({ metadata, expected, output, tags });
|
|
9056
9081
|
const rowId = id || uuidv42();
|
|
@@ -9062,6 +9087,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9062
9087
|
metadata,
|
|
9063
9088
|
tags,
|
|
9064
9089
|
output,
|
|
9090
|
+
origin,
|
|
9065
9091
|
isMerge: false
|
|
9066
9092
|
})
|
|
9067
9093
|
);
|
|
@@ -26019,6 +26045,10 @@ function configureBrowser() {
|
|
|
26019
26045
|
}
|
|
26020
26046
|
return process.env[name];
|
|
26021
26047
|
};
|
|
26048
|
+
isomorph_default.getBraintrustApiKey = async () => {
|
|
26049
|
+
const value = isomorph_default.getEnv("BRAINTRUST_API_KEY");
|
|
26050
|
+
return value?.trim() ? value : void 0;
|
|
26051
|
+
};
|
|
26022
26052
|
isomorph_default.hash = (data) => {
|
|
26023
26053
|
let hash = 0;
|
|
26024
26054
|
for (let i = 0; i < data.length; i++) {
|
|
@@ -26054,6 +26084,7 @@ __export(exports_exports, {
|
|
|
26054
26084
|
DEFAULT_FETCH_BATCH_SIZE: () => DEFAULT_FETCH_BATCH_SIZE,
|
|
26055
26085
|
DEFAULT_MAX_REQUEST_SIZE: () => DEFAULT_MAX_REQUEST_SIZE,
|
|
26056
26086
|
Dataset: () => Dataset2,
|
|
26087
|
+
DatasetPipeline: () => DatasetPipeline,
|
|
26057
26088
|
ERR_PERMALINK: () => ERR_PERMALINK,
|
|
26058
26089
|
Eval: () => Eval,
|
|
26059
26090
|
EvalResultWithSummary: () => EvalResultWithSummary,
|
|
@@ -29312,10 +29343,12 @@ function formatExperimentSummary(summary) {
|
|
|
29312
29343
|
// src/wrappers/shared/flush.ts
|
|
29313
29344
|
async function summarizeAndFlush(experiment, options) {
|
|
29314
29345
|
const shouldDisplay = options.displaySummary ?? true;
|
|
29315
|
-
|
|
29316
|
-
|
|
29317
|
-
|
|
29346
|
+
if (!shouldDisplay) {
|
|
29347
|
+
await experiment.flush();
|
|
29348
|
+
return;
|
|
29318
29349
|
}
|
|
29350
|
+
const summary = await experiment.summarize();
|
|
29351
|
+
console.log(formatExperimentSummary(summary));
|
|
29319
29352
|
}
|
|
29320
29353
|
|
|
29321
29354
|
// src/wrappers/vitest/flush-manager.ts
|
|
@@ -31179,8 +31212,12 @@ var waterfall$1 = awaitify(waterfall);
|
|
|
31179
31212
|
|
|
31180
31213
|
// src/trace.ts
|
|
31181
31214
|
var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
31182
|
-
constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter) {
|
|
31183
|
-
const filterExpr = _SpanFetcher.buildFilter(
|
|
31215
|
+
constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter, includeScorers = false) {
|
|
31216
|
+
const filterExpr = _SpanFetcher.buildFilter(
|
|
31217
|
+
rootSpanId,
|
|
31218
|
+
spanTypeFilter,
|
|
31219
|
+
includeScorers
|
|
31220
|
+
);
|
|
31184
31221
|
super(objectType, void 0, void 0, {
|
|
31185
31222
|
filter: filterExpr
|
|
31186
31223
|
});
|
|
@@ -31189,16 +31226,17 @@ var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
|
31189
31226
|
this._state = _state;
|
|
31190
31227
|
this.spanTypeFilter = spanTypeFilter;
|
|
31191
31228
|
}
|
|
31192
|
-
static buildFilter(rootSpanId, spanTypeFilter) {
|
|
31229
|
+
static buildFilter(rootSpanId, spanTypeFilter, includeScorers = false) {
|
|
31193
31230
|
const children = [
|
|
31194
31231
|
// Base filter: root_span_id = 'value'
|
|
31195
31232
|
{
|
|
31196
31233
|
op: "eq",
|
|
31197
31234
|
left: { op: "ident", name: ["root_span_id"] },
|
|
31198
31235
|
right: { op: "literal", value: rootSpanId }
|
|
31199
|
-
}
|
|
31200
|
-
|
|
31201
|
-
|
|
31236
|
+
}
|
|
31237
|
+
];
|
|
31238
|
+
if (!includeScorers) {
|
|
31239
|
+
children.push({
|
|
31202
31240
|
op: "or",
|
|
31203
31241
|
children: [
|
|
31204
31242
|
{
|
|
@@ -31211,8 +31249,8 @@ var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
|
31211
31249
|
right: { op: "literal", value: "scorer" }
|
|
31212
31250
|
}
|
|
31213
31251
|
]
|
|
31214
|
-
}
|
|
31215
|
-
|
|
31252
|
+
});
|
|
31253
|
+
}
|
|
31216
31254
|
if (spanTypeFilter && spanTypeFilter.length > 0) {
|
|
31217
31255
|
children.push({
|
|
31218
31256
|
op: "in",
|
|
@@ -31238,35 +31276,49 @@ var CachedSpanFetcher = class {
|
|
|
31238
31276
|
fetchFn;
|
|
31239
31277
|
constructor(objectTypeOrFetchFn, objectId, rootSpanId, getState) {
|
|
31240
31278
|
if (typeof objectTypeOrFetchFn === "function") {
|
|
31241
|
-
this.fetchFn = objectTypeOrFetchFn;
|
|
31279
|
+
this.fetchFn = (spanType) => objectTypeOrFetchFn(spanType);
|
|
31242
31280
|
} else {
|
|
31243
31281
|
const objectType = objectTypeOrFetchFn;
|
|
31244
|
-
this.fetchFn = async (spanType) => {
|
|
31282
|
+
this.fetchFn = async (spanType, includeScorers) => {
|
|
31245
31283
|
const state = await getState();
|
|
31246
31284
|
const fetcher = new SpanFetcher(
|
|
31247
31285
|
objectType,
|
|
31248
31286
|
objectId,
|
|
31249
31287
|
rootSpanId,
|
|
31250
31288
|
state,
|
|
31251
|
-
spanType
|
|
31289
|
+
spanType,
|
|
31290
|
+
includeScorers
|
|
31252
31291
|
);
|
|
31253
31292
|
const rows = await fetcher.fetchedData();
|
|
31254
|
-
return rows.
|
|
31293
|
+
return rows.map((row) => ({
|
|
31255
31294
|
input: row.input,
|
|
31256
31295
|
output: row.output,
|
|
31296
|
+
expected: row.expected,
|
|
31297
|
+
error: row.error,
|
|
31298
|
+
scores: row.scores,
|
|
31299
|
+
metrics: row.metrics,
|
|
31257
31300
|
metadata: row.metadata,
|
|
31258
31301
|
span_id: row.span_id,
|
|
31259
31302
|
span_parents: row.span_parents,
|
|
31303
|
+
is_root: row.is_root,
|
|
31260
31304
|
span_attributes: row.span_attributes,
|
|
31261
31305
|
id: row.id,
|
|
31262
31306
|
_xact_id: row._xact_id,
|
|
31263
31307
|
_pagination_key: row._pagination_key,
|
|
31264
|
-
root_span_id: row.root_span_id
|
|
31308
|
+
root_span_id: row.root_span_id,
|
|
31309
|
+
created: row.created,
|
|
31310
|
+
tags: row.tags
|
|
31265
31311
|
}));
|
|
31266
31312
|
};
|
|
31267
31313
|
}
|
|
31268
31314
|
}
|
|
31269
|
-
async getSpans({
|
|
31315
|
+
async getSpans({
|
|
31316
|
+
spanType,
|
|
31317
|
+
includeScorers = false
|
|
31318
|
+
} = {}) {
|
|
31319
|
+
if (includeScorers) {
|
|
31320
|
+
return this.fetchFn(spanType, true);
|
|
31321
|
+
}
|
|
31270
31322
|
if (this.allFetched) {
|
|
31271
31323
|
return this.getFromCache(spanType);
|
|
31272
31324
|
}
|
|
@@ -31283,7 +31335,7 @@ var CachedSpanFetcher = class {
|
|
|
31283
31335
|
return this.getFromCache(spanType);
|
|
31284
31336
|
}
|
|
31285
31337
|
async fetchSpans(spanType) {
|
|
31286
|
-
const spans = await this.fetchFn(spanType);
|
|
31338
|
+
const spans = await this.fetchFn(spanType, false);
|
|
31287
31339
|
for (const span of spans) {
|
|
31288
31340
|
const type = span.span_attributes?.type ?? "";
|
|
31289
31341
|
const existing = this.spanCache.get(type) ?? [];
|
|
@@ -31361,10 +31413,13 @@ var LocalTrace = class {
|
|
|
31361
31413
|
* First checks the local span cache for recently logged spans, then falls
|
|
31362
31414
|
* back to CachedSpanFetcher which handles BTQL fetching and caching.
|
|
31363
31415
|
*/
|
|
31364
|
-
async getSpans({
|
|
31416
|
+
async getSpans({
|
|
31417
|
+
spanType,
|
|
31418
|
+
includeScorers = false
|
|
31419
|
+
} = {}) {
|
|
31365
31420
|
const cachedSpans = this.state.spanCache.getByRootSpanId(this.rootSpanId);
|
|
31366
31421
|
if (cachedSpans && cachedSpans.length > 0) {
|
|
31367
|
-
let spans = cachedSpans.filter(
|
|
31422
|
+
let spans = includeScorers ? cachedSpans : cachedSpans.filter(
|
|
31368
31423
|
(span) => span.span_attributes?.purpose !== "scorer"
|
|
31369
31424
|
);
|
|
31370
31425
|
if (spanType && spanType.length > 0) {
|
|
@@ -31375,13 +31430,19 @@ var LocalTrace = class {
|
|
|
31375
31430
|
return spans.map((span) => ({
|
|
31376
31431
|
input: span.input,
|
|
31377
31432
|
output: span.output,
|
|
31433
|
+
expected: span.expected,
|
|
31434
|
+
error: span.error,
|
|
31435
|
+
scores: span.scores,
|
|
31436
|
+
metrics: span.metrics,
|
|
31378
31437
|
metadata: span.metadata,
|
|
31379
31438
|
span_id: span.span_id,
|
|
31380
31439
|
span_parents: span.span_parents,
|
|
31381
|
-
|
|
31440
|
+
is_root: span.is_root,
|
|
31441
|
+
span_attributes: span.span_attributes,
|
|
31442
|
+
tags: span.tags
|
|
31382
31443
|
}));
|
|
31383
31444
|
}
|
|
31384
|
-
return this.cachedFetcher.getSpans({ spanType });
|
|
31445
|
+
return this.cachedFetcher.getSpans({ spanType, includeScorers });
|
|
31385
31446
|
}
|
|
31386
31447
|
/**
|
|
31387
31448
|
* Get the thread (preprocessed messages) for this trace.
|
|
@@ -32570,6 +32631,34 @@ var defaultReporter = {
|
|
|
32570
32631
|
}
|
|
32571
32632
|
};
|
|
32572
32633
|
|
|
32634
|
+
// src/dataset-pipeline.ts
|
|
32635
|
+
function DatasetPipeline(definition) {
|
|
32636
|
+
if (!globalThis.__braintrust_dataset_pipelines) {
|
|
32637
|
+
globalThis.__braintrust_dataset_pipelines = [];
|
|
32638
|
+
}
|
|
32639
|
+
const storedDefinition = {
|
|
32640
|
+
name: definition.name,
|
|
32641
|
+
source: {
|
|
32642
|
+
projectId: definition.source.projectId,
|
|
32643
|
+
projectName: definition.source.projectName,
|
|
32644
|
+
orgName: definition.source.orgName,
|
|
32645
|
+
filter: definition.source.filter,
|
|
32646
|
+
scope: definition.source.scope ?? "span"
|
|
32647
|
+
},
|
|
32648
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions, @typescript-eslint/no-explicit-any
|
|
32649
|
+
transform: definition.transform,
|
|
32650
|
+
target: {
|
|
32651
|
+
projectId: definition.target.projectId,
|
|
32652
|
+
projectName: definition.target.projectName,
|
|
32653
|
+
orgName: definition.target.orgName,
|
|
32654
|
+
datasetName: definition.target.datasetName,
|
|
32655
|
+
description: definition.target.description,
|
|
32656
|
+
metadata: definition.target.metadata
|
|
32657
|
+
}
|
|
32658
|
+
};
|
|
32659
|
+
globalThis.__braintrust_dataset_pipelines.push(storedDefinition);
|
|
32660
|
+
}
|
|
32661
|
+
|
|
32573
32662
|
// src/framework2.ts
|
|
32574
32663
|
import { z as z12 } from "zod/v3";
|
|
32575
32664
|
var currentFilename = typeof __filename !== "undefined" ? __filename : "unknown";
|
|
@@ -33109,6 +33198,7 @@ export {
|
|
|
33109
33198
|
DEFAULT_FETCH_BATCH_SIZE,
|
|
33110
33199
|
DEFAULT_MAX_REQUEST_SIZE,
|
|
33111
33200
|
Dataset2 as Dataset,
|
|
33201
|
+
DatasetPipeline,
|
|
33112
33202
|
ERR_PERMALINK,
|
|
33113
33203
|
Eval,
|
|
33114
33204
|
EvalResultWithSummary,
|
|
@@ -175,6 +175,7 @@ var iso = {
|
|
|
175
175
|
getRepoInfo: async (_settings) => void 0,
|
|
176
176
|
getPastNAncestors: async () => [],
|
|
177
177
|
getEnv: (_name) => void 0,
|
|
178
|
+
getBraintrustApiKey: async () => void 0,
|
|
178
179
|
getCallerLocation: () => void 0,
|
|
179
180
|
newAsyncLocalStorage: () => new DefaultAsyncLocalStorage(),
|
|
180
181
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -175,6 +175,7 @@ var iso = {
|
|
|
175
175
|
getRepoInfo: async (_settings) => void 0,
|
|
176
176
|
getPastNAncestors: async () => [],
|
|
177
177
|
getEnv: (_name) => void 0,
|
|
178
|
+
getBraintrustApiKey: async () => void 0,
|
|
178
179
|
getCallerLocation: () => void 0,
|
|
179
180
|
newAsyncLocalStorage: () => new DefaultAsyncLocalStorage(),
|
|
180
181
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|