braintrust 3.12.0 → 3.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dev/dist/index.d.mts +25 -6
- package/dev/dist/index.d.ts +25 -6
- package/dev/dist/index.js +134 -43
- package/dev/dist/index.mjs +113 -22
- package/dist/apply-auto-instrumentation.js +170 -170
- package/dist/apply-auto-instrumentation.mjs +1 -1
- package/dist/auto-instrumentations/bundler/esbuild.cjs +1 -0
- package/dist/auto-instrumentations/bundler/esbuild.mjs +2 -2
- package/dist/auto-instrumentations/bundler/next.cjs +1 -0
- package/dist/auto-instrumentations/bundler/next.mjs +3 -3
- package/dist/auto-instrumentations/bundler/rollup.cjs +1 -0
- package/dist/auto-instrumentations/bundler/rollup.mjs +2 -2
- package/dist/auto-instrumentations/bundler/vite.cjs +1 -0
- package/dist/auto-instrumentations/bundler/vite.mjs +2 -2
- package/dist/auto-instrumentations/bundler/webpack-loader.cjs +1 -0
- package/dist/auto-instrumentations/bundler/webpack.cjs +1 -0
- package/dist/auto-instrumentations/bundler/webpack.mjs +3 -3
- package/dist/auto-instrumentations/{chunk-2DPA74KK.mjs → chunk-E5DUYJWK.mjs} +1 -0
- package/dist/auto-instrumentations/{chunk-73BZUKVI.mjs → chunk-GJOO4ESL.mjs} +1 -1
- package/dist/auto-instrumentations/{chunk-AFXRW7I7.mjs → chunk-WFEUJACP.mjs} +1 -1
- package/dist/auto-instrumentations/hook.mjs +1 -0
- package/dist/auto-instrumentations/index.cjs +1 -0
- package/dist/auto-instrumentations/index.mjs +1 -1
- package/dist/browser.d.mts +149 -21
- package/dist/browser.d.ts +149 -21
- package/dist/browser.js +114 -24
- package/dist/browser.mjs +114 -24
- package/dist/{chunk-BW4DF4CY.js → chunk-26JGOELH.js} +1 -0
- package/dist/{chunk-MSLBGITU.mjs → chunk-75IQCUB2.mjs} +1 -0
- package/dist/cli.js +121 -44
- package/dist/edge-light.d.mts +1 -1
- package/dist/edge-light.d.ts +1 -1
- package/dist/edge-light.js +114 -24
- package/dist/edge-light.mjs +114 -24
- package/dist/index.d.mts +149 -21
- package/dist/index.d.ts +149 -21
- package/dist/index.js +529 -394
- package/dist/index.mjs +161 -26
- package/dist/instrumentation/index.d.mts +40 -3
- package/dist/instrumentation/index.d.ts +40 -3
- package/dist/instrumentation/index.js +15 -2
- package/dist/instrumentation/index.mjs +15 -2
- package/dist/workerd.d.mts +1 -1
- package/dist/workerd.d.ts +1 -1
- package/dist/workerd.js +114 -24
- package/dist/workerd.mjs +114 -24
- package/package.json +3 -17
- package/util/dist/index.d.mts +3 -1
- package/util/dist/index.d.ts +3 -1
package/dist/edge-light.js
CHANGED
|
@@ -47,6 +47,7 @@ __export(edge_light_exports, {
|
|
|
47
47
|
DEFAULT_FETCH_BATCH_SIZE: () => DEFAULT_FETCH_BATCH_SIZE,
|
|
48
48
|
DEFAULT_MAX_REQUEST_SIZE: () => DEFAULT_MAX_REQUEST_SIZE,
|
|
49
49
|
Dataset: () => Dataset2,
|
|
50
|
+
DatasetPipeline: () => DatasetPipeline,
|
|
50
51
|
ERR_PERMALINK: () => ERR_PERMALINK,
|
|
51
52
|
Eval: () => Eval,
|
|
52
53
|
EvalResultWithSummary: () => EvalResultWithSummary,
|
|
@@ -267,6 +268,7 @@ var iso = {
|
|
|
267
268
|
getRepoInfo: async (_settings) => void 0,
|
|
268
269
|
getPastNAncestors: async () => [],
|
|
269
270
|
getEnv: (_name) => void 0,
|
|
271
|
+
getBraintrustApiKey: async () => void 0,
|
|
270
272
|
getCallerLocation: () => void 0,
|
|
271
273
|
newAsyncLocalStorage: () => new DefaultAsyncLocalStorage(),
|
|
272
274
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -5606,6 +5608,19 @@ var JSONAttachment = class extends Attachment {
|
|
|
5606
5608
|
*/
|
|
5607
5609
|
constructor(data, options) {
|
|
5608
5610
|
const { filename = "data.json", pretty = false, state } = options ?? {};
|
|
5611
|
+
const deferredJsonAttachment = globalThis.__BT_DATASET_PIPELINE_DEFER_JSON_ATTACHMENT__;
|
|
5612
|
+
if (deferredJsonAttachment) {
|
|
5613
|
+
super({
|
|
5614
|
+
data: new Blob([]),
|
|
5615
|
+
filename,
|
|
5616
|
+
contentType: "application/json",
|
|
5617
|
+
state
|
|
5618
|
+
});
|
|
5619
|
+
return deferredJsonAttachment(data, {
|
|
5620
|
+
filename,
|
|
5621
|
+
pretty
|
|
5622
|
+
});
|
|
5623
|
+
}
|
|
5609
5624
|
const jsonString = pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data);
|
|
5610
5625
|
const blob = new Blob([jsonString], { type: "application/json" });
|
|
5611
5626
|
super({
|
|
@@ -7598,10 +7613,11 @@ async function login(options = {}) {
|
|
|
7598
7613
|
async function loginToState(options = {}) {
|
|
7599
7614
|
const {
|
|
7600
7615
|
appUrl = isomorph_default.getEnv("BRAINTRUST_APP_URL") || "https://www.braintrust.dev",
|
|
7601
|
-
apiKey
|
|
7616
|
+
apiKey: apiKeyArg,
|
|
7602
7617
|
orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME"),
|
|
7603
7618
|
fetch: fetch2 = globalThis.fetch
|
|
7604
7619
|
} = options || {};
|
|
7620
|
+
const apiKey = apiKeyArg !== void 0 ? apiKeyArg : await isomorph_default.getBraintrustApiKey();
|
|
7605
7621
|
const appPublicUrl = isomorph_default.getEnv("BRAINTRUST_APP_PUBLIC_URL") || appUrl;
|
|
7606
7622
|
const state = new BraintrustState(options);
|
|
7607
7623
|
state.resetLoginInfo();
|
|
@@ -8842,9 +8858,15 @@ var SpanImpl = class _SpanImpl {
|
|
|
8842
8858
|
const cachedSpan = {
|
|
8843
8859
|
input: partialRecord.input,
|
|
8844
8860
|
output: partialRecord.output,
|
|
8861
|
+
expected: partialRecord.expected,
|
|
8862
|
+
error: partialRecord.error,
|
|
8863
|
+
scores: partialRecord.scores,
|
|
8864
|
+
metrics: partialRecord.metrics,
|
|
8845
8865
|
metadata: partialRecord.metadata,
|
|
8866
|
+
tags: partialRecord.tags,
|
|
8846
8867
|
span_id: this._spanId,
|
|
8847
8868
|
span_parents: this._spanParents,
|
|
8869
|
+
is_root: this._spanId === this._rootSpanId,
|
|
8848
8870
|
span_attributes: partialRecord.span_attributes
|
|
8849
8871
|
};
|
|
8850
8872
|
this._state.spanCache.queueWrite(
|
|
@@ -9180,6 +9202,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9180
9202
|
metadata,
|
|
9181
9203
|
tags,
|
|
9182
9204
|
output,
|
|
9205
|
+
origin,
|
|
9183
9206
|
isMerge
|
|
9184
9207
|
}) {
|
|
9185
9208
|
return new LazyValue(async () => {
|
|
@@ -9194,6 +9217,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9194
9217
|
created: !isMerge ? (/* @__PURE__ */ new Date()).toISOString() : void 0,
|
|
9195
9218
|
//if we're merging/updating an event we will not add this ts
|
|
9196
9219
|
metadata,
|
|
9220
|
+
origin,
|
|
9197
9221
|
...!!isMerge ? {
|
|
9198
9222
|
[IS_MERGE_FIELD]: true
|
|
9199
9223
|
} : {}
|
|
@@ -9213,6 +9237,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9213
9237
|
* about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
|
|
9214
9238
|
* `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
|
|
9215
9239
|
* JSON-serializable type, but its keys must be strings.
|
|
9240
|
+
* @param event.origin (Optional) a reference to the source object this dataset record was derived from.
|
|
9216
9241
|
* @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
|
|
9217
9242
|
* @param event.output: (Deprecated) The output of your application. Use `expected` instead.
|
|
9218
9243
|
* @returns The `id` of the logged record.
|
|
@@ -9223,7 +9248,8 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9223
9248
|
metadata,
|
|
9224
9249
|
tags,
|
|
9225
9250
|
id,
|
|
9226
|
-
output
|
|
9251
|
+
output,
|
|
9252
|
+
origin
|
|
9227
9253
|
}) {
|
|
9228
9254
|
this.validateEvent({ metadata, expected, output, tags });
|
|
9229
9255
|
const rowId = id || (0, import_uuid2.v4)();
|
|
@@ -9235,6 +9261,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9235
9261
|
metadata,
|
|
9236
9262
|
tags,
|
|
9237
9263
|
output,
|
|
9264
|
+
origin,
|
|
9238
9265
|
isMerge: false
|
|
9239
9266
|
})
|
|
9240
9267
|
);
|
|
@@ -26230,6 +26257,10 @@ function configureEdgeLight() {
|
|
|
26230
26257
|
}
|
|
26231
26258
|
return process.env[name];
|
|
26232
26259
|
};
|
|
26260
|
+
isomorph_default.getBraintrustApiKey = async () => {
|
|
26261
|
+
const value = isomorph_default.getEnv("BRAINTRUST_API_KEY");
|
|
26262
|
+
return value?.trim() ? value : void 0;
|
|
26263
|
+
};
|
|
26233
26264
|
isomorph_default.hash = (data) => {
|
|
26234
26265
|
let hash = 0;
|
|
26235
26266
|
for (let i = 0; i < data.length; i++) {
|
|
@@ -26265,6 +26296,7 @@ __export(exports_exports, {
|
|
|
26265
26296
|
DEFAULT_FETCH_BATCH_SIZE: () => DEFAULT_FETCH_BATCH_SIZE,
|
|
26266
26297
|
DEFAULT_MAX_REQUEST_SIZE: () => DEFAULT_MAX_REQUEST_SIZE,
|
|
26267
26298
|
Dataset: () => Dataset2,
|
|
26299
|
+
DatasetPipeline: () => DatasetPipeline,
|
|
26268
26300
|
ERR_PERMALINK: () => ERR_PERMALINK,
|
|
26269
26301
|
Eval: () => Eval,
|
|
26270
26302
|
EvalResultWithSummary: () => EvalResultWithSummary,
|
|
@@ -29523,10 +29555,12 @@ function formatExperimentSummary(summary) {
|
|
|
29523
29555
|
// src/wrappers/shared/flush.ts
|
|
29524
29556
|
async function summarizeAndFlush(experiment, options) {
|
|
29525
29557
|
const shouldDisplay = options.displaySummary ?? true;
|
|
29526
|
-
|
|
29527
|
-
|
|
29528
|
-
|
|
29558
|
+
if (!shouldDisplay) {
|
|
29559
|
+
await experiment.flush();
|
|
29560
|
+
return;
|
|
29529
29561
|
}
|
|
29562
|
+
const summary = await experiment.summarize();
|
|
29563
|
+
console.log(formatExperimentSummary(summary));
|
|
29530
29564
|
}
|
|
29531
29565
|
|
|
29532
29566
|
// src/wrappers/vitest/flush-manager.ts
|
|
@@ -31390,8 +31424,12 @@ var waterfall$1 = awaitify(waterfall);
|
|
|
31390
31424
|
|
|
31391
31425
|
// src/trace.ts
|
|
31392
31426
|
var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
31393
|
-
constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter) {
|
|
31394
|
-
const filterExpr = _SpanFetcher.buildFilter(
|
|
31427
|
+
constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter, includeScorers = false) {
|
|
31428
|
+
const filterExpr = _SpanFetcher.buildFilter(
|
|
31429
|
+
rootSpanId,
|
|
31430
|
+
spanTypeFilter,
|
|
31431
|
+
includeScorers
|
|
31432
|
+
);
|
|
31395
31433
|
super(objectType, void 0, void 0, {
|
|
31396
31434
|
filter: filterExpr
|
|
31397
31435
|
});
|
|
@@ -31400,16 +31438,17 @@ var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
|
31400
31438
|
this._state = _state;
|
|
31401
31439
|
this.spanTypeFilter = spanTypeFilter;
|
|
31402
31440
|
}
|
|
31403
|
-
static buildFilter(rootSpanId, spanTypeFilter) {
|
|
31441
|
+
static buildFilter(rootSpanId, spanTypeFilter, includeScorers = false) {
|
|
31404
31442
|
const children = [
|
|
31405
31443
|
// Base filter: root_span_id = 'value'
|
|
31406
31444
|
{
|
|
31407
31445
|
op: "eq",
|
|
31408
31446
|
left: { op: "ident", name: ["root_span_id"] },
|
|
31409
31447
|
right: { op: "literal", value: rootSpanId }
|
|
31410
|
-
}
|
|
31411
|
-
|
|
31412
|
-
|
|
31448
|
+
}
|
|
31449
|
+
];
|
|
31450
|
+
if (!includeScorers) {
|
|
31451
|
+
children.push({
|
|
31413
31452
|
op: "or",
|
|
31414
31453
|
children: [
|
|
31415
31454
|
{
|
|
@@ -31422,8 +31461,8 @@ var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
|
31422
31461
|
right: { op: "literal", value: "scorer" }
|
|
31423
31462
|
}
|
|
31424
31463
|
]
|
|
31425
|
-
}
|
|
31426
|
-
|
|
31464
|
+
});
|
|
31465
|
+
}
|
|
31427
31466
|
if (spanTypeFilter && spanTypeFilter.length > 0) {
|
|
31428
31467
|
children.push({
|
|
31429
31468
|
op: "in",
|
|
@@ -31449,35 +31488,49 @@ var CachedSpanFetcher = class {
|
|
|
31449
31488
|
fetchFn;
|
|
31450
31489
|
constructor(objectTypeOrFetchFn, objectId, rootSpanId, getState) {
|
|
31451
31490
|
if (typeof objectTypeOrFetchFn === "function") {
|
|
31452
|
-
this.fetchFn = objectTypeOrFetchFn;
|
|
31491
|
+
this.fetchFn = (spanType) => objectTypeOrFetchFn(spanType);
|
|
31453
31492
|
} else {
|
|
31454
31493
|
const objectType = objectTypeOrFetchFn;
|
|
31455
|
-
this.fetchFn = async (spanType) => {
|
|
31494
|
+
this.fetchFn = async (spanType, includeScorers) => {
|
|
31456
31495
|
const state = await getState();
|
|
31457
31496
|
const fetcher = new SpanFetcher(
|
|
31458
31497
|
objectType,
|
|
31459
31498
|
objectId,
|
|
31460
31499
|
rootSpanId,
|
|
31461
31500
|
state,
|
|
31462
|
-
spanType
|
|
31501
|
+
spanType,
|
|
31502
|
+
includeScorers
|
|
31463
31503
|
);
|
|
31464
31504
|
const rows = await fetcher.fetchedData();
|
|
31465
|
-
return rows.
|
|
31505
|
+
return rows.map((row) => ({
|
|
31466
31506
|
input: row.input,
|
|
31467
31507
|
output: row.output,
|
|
31508
|
+
expected: row.expected,
|
|
31509
|
+
error: row.error,
|
|
31510
|
+
scores: row.scores,
|
|
31511
|
+
metrics: row.metrics,
|
|
31468
31512
|
metadata: row.metadata,
|
|
31469
31513
|
span_id: row.span_id,
|
|
31470
31514
|
span_parents: row.span_parents,
|
|
31515
|
+
is_root: row.is_root,
|
|
31471
31516
|
span_attributes: row.span_attributes,
|
|
31472
31517
|
id: row.id,
|
|
31473
31518
|
_xact_id: row._xact_id,
|
|
31474
31519
|
_pagination_key: row._pagination_key,
|
|
31475
|
-
root_span_id: row.root_span_id
|
|
31520
|
+
root_span_id: row.root_span_id,
|
|
31521
|
+
created: row.created,
|
|
31522
|
+
tags: row.tags
|
|
31476
31523
|
}));
|
|
31477
31524
|
};
|
|
31478
31525
|
}
|
|
31479
31526
|
}
|
|
31480
|
-
async getSpans({
|
|
31527
|
+
async getSpans({
|
|
31528
|
+
spanType,
|
|
31529
|
+
includeScorers = false
|
|
31530
|
+
} = {}) {
|
|
31531
|
+
if (includeScorers) {
|
|
31532
|
+
return this.fetchFn(spanType, true);
|
|
31533
|
+
}
|
|
31481
31534
|
if (this.allFetched) {
|
|
31482
31535
|
return this.getFromCache(spanType);
|
|
31483
31536
|
}
|
|
@@ -31494,7 +31547,7 @@ var CachedSpanFetcher = class {
|
|
|
31494
31547
|
return this.getFromCache(spanType);
|
|
31495
31548
|
}
|
|
31496
31549
|
async fetchSpans(spanType) {
|
|
31497
|
-
const spans = await this.fetchFn(spanType);
|
|
31550
|
+
const spans = await this.fetchFn(spanType, false);
|
|
31498
31551
|
for (const span of spans) {
|
|
31499
31552
|
const type = span.span_attributes?.type ?? "";
|
|
31500
31553
|
const existing = this.spanCache.get(type) ?? [];
|
|
@@ -31572,10 +31625,13 @@ var LocalTrace = class {
|
|
|
31572
31625
|
* First checks the local span cache for recently logged spans, then falls
|
|
31573
31626
|
* back to CachedSpanFetcher which handles BTQL fetching and caching.
|
|
31574
31627
|
*/
|
|
31575
|
-
async getSpans({
|
|
31628
|
+
async getSpans({
|
|
31629
|
+
spanType,
|
|
31630
|
+
includeScorers = false
|
|
31631
|
+
} = {}) {
|
|
31576
31632
|
const cachedSpans = this.state.spanCache.getByRootSpanId(this.rootSpanId);
|
|
31577
31633
|
if (cachedSpans && cachedSpans.length > 0) {
|
|
31578
|
-
let spans = cachedSpans.filter(
|
|
31634
|
+
let spans = includeScorers ? cachedSpans : cachedSpans.filter(
|
|
31579
31635
|
(span) => span.span_attributes?.purpose !== "scorer"
|
|
31580
31636
|
);
|
|
31581
31637
|
if (spanType && spanType.length > 0) {
|
|
@@ -31586,13 +31642,19 @@ var LocalTrace = class {
|
|
|
31586
31642
|
return spans.map((span) => ({
|
|
31587
31643
|
input: span.input,
|
|
31588
31644
|
output: span.output,
|
|
31645
|
+
expected: span.expected,
|
|
31646
|
+
error: span.error,
|
|
31647
|
+
scores: span.scores,
|
|
31648
|
+
metrics: span.metrics,
|
|
31589
31649
|
metadata: span.metadata,
|
|
31590
31650
|
span_id: span.span_id,
|
|
31591
31651
|
span_parents: span.span_parents,
|
|
31592
|
-
|
|
31652
|
+
is_root: span.is_root,
|
|
31653
|
+
span_attributes: span.span_attributes,
|
|
31654
|
+
tags: span.tags
|
|
31593
31655
|
}));
|
|
31594
31656
|
}
|
|
31595
|
-
return this.cachedFetcher.getSpans({ spanType });
|
|
31657
|
+
return this.cachedFetcher.getSpans({ spanType, includeScorers });
|
|
31596
31658
|
}
|
|
31597
31659
|
/**
|
|
31598
31660
|
* Get the thread (preprocessed messages) for this trace.
|
|
@@ -32781,6 +32843,34 @@ var defaultReporter = {
|
|
|
32781
32843
|
}
|
|
32782
32844
|
};
|
|
32783
32845
|
|
|
32846
|
+
// src/dataset-pipeline.ts
|
|
32847
|
+
function DatasetPipeline(definition) {
|
|
32848
|
+
if (!globalThis.__braintrust_dataset_pipelines) {
|
|
32849
|
+
globalThis.__braintrust_dataset_pipelines = [];
|
|
32850
|
+
}
|
|
32851
|
+
const storedDefinition = {
|
|
32852
|
+
name: definition.name,
|
|
32853
|
+
source: {
|
|
32854
|
+
projectId: definition.source.projectId,
|
|
32855
|
+
projectName: definition.source.projectName,
|
|
32856
|
+
orgName: definition.source.orgName,
|
|
32857
|
+
filter: definition.source.filter,
|
|
32858
|
+
scope: definition.source.scope ?? "span"
|
|
32859
|
+
},
|
|
32860
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions, @typescript-eslint/no-explicit-any
|
|
32861
|
+
transform: definition.transform,
|
|
32862
|
+
target: {
|
|
32863
|
+
projectId: definition.target.projectId,
|
|
32864
|
+
projectName: definition.target.projectName,
|
|
32865
|
+
orgName: definition.target.orgName,
|
|
32866
|
+
datasetName: definition.target.datasetName,
|
|
32867
|
+
description: definition.target.description,
|
|
32868
|
+
metadata: definition.target.metadata
|
|
32869
|
+
}
|
|
32870
|
+
};
|
|
32871
|
+
globalThis.__braintrust_dataset_pipelines.push(storedDefinition);
|
|
32872
|
+
}
|
|
32873
|
+
|
|
32784
32874
|
// src/framework2.ts
|
|
32785
32875
|
var import_v312 = require("zod/v3");
|
|
32786
32876
|
var currentFilename = typeof __filename !== "undefined" ? __filename : "unknown";
|
package/dist/edge-light.mjs
CHANGED
|
@@ -89,6 +89,7 @@ var iso = {
|
|
|
89
89
|
getRepoInfo: async (_settings) => void 0,
|
|
90
90
|
getPastNAncestors: async () => [],
|
|
91
91
|
getEnv: (_name) => void 0,
|
|
92
|
+
getBraintrustApiKey: async () => void 0,
|
|
92
93
|
getCallerLocation: () => void 0,
|
|
93
94
|
newAsyncLocalStorage: () => new DefaultAsyncLocalStorage(),
|
|
94
95
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -5430,6 +5431,19 @@ var JSONAttachment = class extends Attachment {
|
|
|
5430
5431
|
*/
|
|
5431
5432
|
constructor(data, options) {
|
|
5432
5433
|
const { filename = "data.json", pretty = false, state } = options ?? {};
|
|
5434
|
+
const deferredJsonAttachment = globalThis.__BT_DATASET_PIPELINE_DEFER_JSON_ATTACHMENT__;
|
|
5435
|
+
if (deferredJsonAttachment) {
|
|
5436
|
+
super({
|
|
5437
|
+
data: new Blob([]),
|
|
5438
|
+
filename,
|
|
5439
|
+
contentType: "application/json",
|
|
5440
|
+
state
|
|
5441
|
+
});
|
|
5442
|
+
return deferredJsonAttachment(data, {
|
|
5443
|
+
filename,
|
|
5444
|
+
pretty
|
|
5445
|
+
});
|
|
5446
|
+
}
|
|
5433
5447
|
const jsonString = pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data);
|
|
5434
5448
|
const blob = new Blob([jsonString], { type: "application/json" });
|
|
5435
5449
|
super({
|
|
@@ -7422,10 +7436,11 @@ async function login(options = {}) {
|
|
|
7422
7436
|
async function loginToState(options = {}) {
|
|
7423
7437
|
const {
|
|
7424
7438
|
appUrl = isomorph_default.getEnv("BRAINTRUST_APP_URL") || "https://www.braintrust.dev",
|
|
7425
|
-
apiKey
|
|
7439
|
+
apiKey: apiKeyArg,
|
|
7426
7440
|
orgName = isomorph_default.getEnv("BRAINTRUST_ORG_NAME"),
|
|
7427
7441
|
fetch: fetch2 = globalThis.fetch
|
|
7428
7442
|
} = options || {};
|
|
7443
|
+
const apiKey = apiKeyArg !== void 0 ? apiKeyArg : await isomorph_default.getBraintrustApiKey();
|
|
7429
7444
|
const appPublicUrl = isomorph_default.getEnv("BRAINTRUST_APP_PUBLIC_URL") || appUrl;
|
|
7430
7445
|
const state = new BraintrustState(options);
|
|
7431
7446
|
state.resetLoginInfo();
|
|
@@ -8666,9 +8681,15 @@ var SpanImpl = class _SpanImpl {
|
|
|
8666
8681
|
const cachedSpan = {
|
|
8667
8682
|
input: partialRecord.input,
|
|
8668
8683
|
output: partialRecord.output,
|
|
8684
|
+
expected: partialRecord.expected,
|
|
8685
|
+
error: partialRecord.error,
|
|
8686
|
+
scores: partialRecord.scores,
|
|
8687
|
+
metrics: partialRecord.metrics,
|
|
8669
8688
|
metadata: partialRecord.metadata,
|
|
8689
|
+
tags: partialRecord.tags,
|
|
8670
8690
|
span_id: this._spanId,
|
|
8671
8691
|
span_parents: this._spanParents,
|
|
8692
|
+
is_root: this._spanId === this._rootSpanId,
|
|
8672
8693
|
span_attributes: partialRecord.span_attributes
|
|
8673
8694
|
};
|
|
8674
8695
|
this._state.spanCache.queueWrite(
|
|
@@ -9004,6 +9025,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9004
9025
|
metadata,
|
|
9005
9026
|
tags,
|
|
9006
9027
|
output,
|
|
9028
|
+
origin,
|
|
9007
9029
|
isMerge
|
|
9008
9030
|
}) {
|
|
9009
9031
|
return new LazyValue(async () => {
|
|
@@ -9018,6 +9040,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9018
9040
|
created: !isMerge ? (/* @__PURE__ */ new Date()).toISOString() : void 0,
|
|
9019
9041
|
//if we're merging/updating an event we will not add this ts
|
|
9020
9042
|
metadata,
|
|
9043
|
+
origin,
|
|
9021
9044
|
...!!isMerge ? {
|
|
9022
9045
|
[IS_MERGE_FIELD]: true
|
|
9023
9046
|
} : {}
|
|
@@ -9037,6 +9060,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9037
9060
|
* about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the
|
|
9038
9061
|
* `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any
|
|
9039
9062
|
* JSON-serializable type, but its keys must be strings.
|
|
9063
|
+
* @param event.origin (Optional) a reference to the source object this dataset record was derived from.
|
|
9040
9064
|
* @param event.id (Optional) a unique identifier for the event. If you don't provide one, Braintrust will generate one for you.
|
|
9041
9065
|
* @param event.output: (Deprecated) The output of your application. Use `expected` instead.
|
|
9042
9066
|
* @returns The `id` of the logged record.
|
|
@@ -9047,7 +9071,8 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9047
9071
|
metadata,
|
|
9048
9072
|
tags,
|
|
9049
9073
|
id,
|
|
9050
|
-
output
|
|
9074
|
+
output,
|
|
9075
|
+
origin
|
|
9051
9076
|
}) {
|
|
9052
9077
|
this.validateEvent({ metadata, expected, output, tags });
|
|
9053
9078
|
const rowId = id || uuidv42();
|
|
@@ -9059,6 +9084,7 @@ var Dataset2 = class extends ObjectFetcher {
|
|
|
9059
9084
|
metadata,
|
|
9060
9085
|
tags,
|
|
9061
9086
|
output,
|
|
9087
|
+
origin,
|
|
9062
9088
|
isMerge: false
|
|
9063
9089
|
})
|
|
9064
9090
|
);
|
|
@@ -26054,6 +26080,10 @@ function configureEdgeLight() {
|
|
|
26054
26080
|
}
|
|
26055
26081
|
return process.env[name];
|
|
26056
26082
|
};
|
|
26083
|
+
isomorph_default.getBraintrustApiKey = async () => {
|
|
26084
|
+
const value = isomorph_default.getEnv("BRAINTRUST_API_KEY");
|
|
26085
|
+
return value?.trim() ? value : void 0;
|
|
26086
|
+
};
|
|
26057
26087
|
isomorph_default.hash = (data) => {
|
|
26058
26088
|
let hash = 0;
|
|
26059
26089
|
for (let i = 0; i < data.length; i++) {
|
|
@@ -26089,6 +26119,7 @@ __export(exports_exports, {
|
|
|
26089
26119
|
DEFAULT_FETCH_BATCH_SIZE: () => DEFAULT_FETCH_BATCH_SIZE,
|
|
26090
26120
|
DEFAULT_MAX_REQUEST_SIZE: () => DEFAULT_MAX_REQUEST_SIZE,
|
|
26091
26121
|
Dataset: () => Dataset2,
|
|
26122
|
+
DatasetPipeline: () => DatasetPipeline,
|
|
26092
26123
|
ERR_PERMALINK: () => ERR_PERMALINK,
|
|
26093
26124
|
Eval: () => Eval,
|
|
26094
26125
|
EvalResultWithSummary: () => EvalResultWithSummary,
|
|
@@ -29347,10 +29378,12 @@ function formatExperimentSummary(summary) {
|
|
|
29347
29378
|
// src/wrappers/shared/flush.ts
|
|
29348
29379
|
async function summarizeAndFlush(experiment, options) {
|
|
29349
29380
|
const shouldDisplay = options.displaySummary ?? true;
|
|
29350
|
-
|
|
29351
|
-
|
|
29352
|
-
|
|
29381
|
+
if (!shouldDisplay) {
|
|
29382
|
+
await experiment.flush();
|
|
29383
|
+
return;
|
|
29353
29384
|
}
|
|
29385
|
+
const summary = await experiment.summarize();
|
|
29386
|
+
console.log(formatExperimentSummary(summary));
|
|
29354
29387
|
}
|
|
29355
29388
|
|
|
29356
29389
|
// src/wrappers/vitest/flush-manager.ts
|
|
@@ -31214,8 +31247,12 @@ var waterfall$1 = awaitify(waterfall);
|
|
|
31214
31247
|
|
|
31215
31248
|
// src/trace.ts
|
|
31216
31249
|
var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
31217
|
-
constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter) {
|
|
31218
|
-
const filterExpr = _SpanFetcher.buildFilter(
|
|
31250
|
+
constructor(objectType, _objectId, rootSpanId, _state, spanTypeFilter, includeScorers = false) {
|
|
31251
|
+
const filterExpr = _SpanFetcher.buildFilter(
|
|
31252
|
+
rootSpanId,
|
|
31253
|
+
spanTypeFilter,
|
|
31254
|
+
includeScorers
|
|
31255
|
+
);
|
|
31219
31256
|
super(objectType, void 0, void 0, {
|
|
31220
31257
|
filter: filterExpr
|
|
31221
31258
|
});
|
|
@@ -31224,16 +31261,17 @@ var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
|
31224
31261
|
this._state = _state;
|
|
31225
31262
|
this.spanTypeFilter = spanTypeFilter;
|
|
31226
31263
|
}
|
|
31227
|
-
static buildFilter(rootSpanId, spanTypeFilter) {
|
|
31264
|
+
static buildFilter(rootSpanId, spanTypeFilter, includeScorers = false) {
|
|
31228
31265
|
const children = [
|
|
31229
31266
|
// Base filter: root_span_id = 'value'
|
|
31230
31267
|
{
|
|
31231
31268
|
op: "eq",
|
|
31232
31269
|
left: { op: "ident", name: ["root_span_id"] },
|
|
31233
31270
|
right: { op: "literal", value: rootSpanId }
|
|
31234
|
-
}
|
|
31235
|
-
|
|
31236
|
-
|
|
31271
|
+
}
|
|
31272
|
+
];
|
|
31273
|
+
if (!includeScorers) {
|
|
31274
|
+
children.push({
|
|
31237
31275
|
op: "or",
|
|
31238
31276
|
children: [
|
|
31239
31277
|
{
|
|
@@ -31246,8 +31284,8 @@ var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
|
31246
31284
|
right: { op: "literal", value: "scorer" }
|
|
31247
31285
|
}
|
|
31248
31286
|
]
|
|
31249
|
-
}
|
|
31250
|
-
|
|
31287
|
+
});
|
|
31288
|
+
}
|
|
31251
31289
|
if (spanTypeFilter && spanTypeFilter.length > 0) {
|
|
31252
31290
|
children.push({
|
|
31253
31291
|
op: "in",
|
|
@@ -31273,35 +31311,49 @@ var CachedSpanFetcher = class {
|
|
|
31273
31311
|
fetchFn;
|
|
31274
31312
|
constructor(objectTypeOrFetchFn, objectId, rootSpanId, getState) {
|
|
31275
31313
|
if (typeof objectTypeOrFetchFn === "function") {
|
|
31276
|
-
this.fetchFn = objectTypeOrFetchFn;
|
|
31314
|
+
this.fetchFn = (spanType) => objectTypeOrFetchFn(spanType);
|
|
31277
31315
|
} else {
|
|
31278
31316
|
const objectType = objectTypeOrFetchFn;
|
|
31279
|
-
this.fetchFn = async (spanType) => {
|
|
31317
|
+
this.fetchFn = async (spanType, includeScorers) => {
|
|
31280
31318
|
const state = await getState();
|
|
31281
31319
|
const fetcher = new SpanFetcher(
|
|
31282
31320
|
objectType,
|
|
31283
31321
|
objectId,
|
|
31284
31322
|
rootSpanId,
|
|
31285
31323
|
state,
|
|
31286
|
-
spanType
|
|
31324
|
+
spanType,
|
|
31325
|
+
includeScorers
|
|
31287
31326
|
);
|
|
31288
31327
|
const rows = await fetcher.fetchedData();
|
|
31289
|
-
return rows.
|
|
31328
|
+
return rows.map((row) => ({
|
|
31290
31329
|
input: row.input,
|
|
31291
31330
|
output: row.output,
|
|
31331
|
+
expected: row.expected,
|
|
31332
|
+
error: row.error,
|
|
31333
|
+
scores: row.scores,
|
|
31334
|
+
metrics: row.metrics,
|
|
31292
31335
|
metadata: row.metadata,
|
|
31293
31336
|
span_id: row.span_id,
|
|
31294
31337
|
span_parents: row.span_parents,
|
|
31338
|
+
is_root: row.is_root,
|
|
31295
31339
|
span_attributes: row.span_attributes,
|
|
31296
31340
|
id: row.id,
|
|
31297
31341
|
_xact_id: row._xact_id,
|
|
31298
31342
|
_pagination_key: row._pagination_key,
|
|
31299
|
-
root_span_id: row.root_span_id
|
|
31343
|
+
root_span_id: row.root_span_id,
|
|
31344
|
+
created: row.created,
|
|
31345
|
+
tags: row.tags
|
|
31300
31346
|
}));
|
|
31301
31347
|
};
|
|
31302
31348
|
}
|
|
31303
31349
|
}
|
|
31304
|
-
async getSpans({
|
|
31350
|
+
async getSpans({
|
|
31351
|
+
spanType,
|
|
31352
|
+
includeScorers = false
|
|
31353
|
+
} = {}) {
|
|
31354
|
+
if (includeScorers) {
|
|
31355
|
+
return this.fetchFn(spanType, true);
|
|
31356
|
+
}
|
|
31305
31357
|
if (this.allFetched) {
|
|
31306
31358
|
return this.getFromCache(spanType);
|
|
31307
31359
|
}
|
|
@@ -31318,7 +31370,7 @@ var CachedSpanFetcher = class {
|
|
|
31318
31370
|
return this.getFromCache(spanType);
|
|
31319
31371
|
}
|
|
31320
31372
|
async fetchSpans(spanType) {
|
|
31321
|
-
const spans = await this.fetchFn(spanType);
|
|
31373
|
+
const spans = await this.fetchFn(spanType, false);
|
|
31322
31374
|
for (const span of spans) {
|
|
31323
31375
|
const type = span.span_attributes?.type ?? "";
|
|
31324
31376
|
const existing = this.spanCache.get(type) ?? [];
|
|
@@ -31396,10 +31448,13 @@ var LocalTrace = class {
|
|
|
31396
31448
|
* First checks the local span cache for recently logged spans, then falls
|
|
31397
31449
|
* back to CachedSpanFetcher which handles BTQL fetching and caching.
|
|
31398
31450
|
*/
|
|
31399
|
-
async getSpans({
|
|
31451
|
+
async getSpans({
|
|
31452
|
+
spanType,
|
|
31453
|
+
includeScorers = false
|
|
31454
|
+
} = {}) {
|
|
31400
31455
|
const cachedSpans = this.state.spanCache.getByRootSpanId(this.rootSpanId);
|
|
31401
31456
|
if (cachedSpans && cachedSpans.length > 0) {
|
|
31402
|
-
let spans = cachedSpans.filter(
|
|
31457
|
+
let spans = includeScorers ? cachedSpans : cachedSpans.filter(
|
|
31403
31458
|
(span) => span.span_attributes?.purpose !== "scorer"
|
|
31404
31459
|
);
|
|
31405
31460
|
if (spanType && spanType.length > 0) {
|
|
@@ -31410,13 +31465,19 @@ var LocalTrace = class {
|
|
|
31410
31465
|
return spans.map((span) => ({
|
|
31411
31466
|
input: span.input,
|
|
31412
31467
|
output: span.output,
|
|
31468
|
+
expected: span.expected,
|
|
31469
|
+
error: span.error,
|
|
31470
|
+
scores: span.scores,
|
|
31471
|
+
metrics: span.metrics,
|
|
31413
31472
|
metadata: span.metadata,
|
|
31414
31473
|
span_id: span.span_id,
|
|
31415
31474
|
span_parents: span.span_parents,
|
|
31416
|
-
|
|
31475
|
+
is_root: span.is_root,
|
|
31476
|
+
span_attributes: span.span_attributes,
|
|
31477
|
+
tags: span.tags
|
|
31417
31478
|
}));
|
|
31418
31479
|
}
|
|
31419
|
-
return this.cachedFetcher.getSpans({ spanType });
|
|
31480
|
+
return this.cachedFetcher.getSpans({ spanType, includeScorers });
|
|
31420
31481
|
}
|
|
31421
31482
|
/**
|
|
31422
31483
|
* Get the thread (preprocessed messages) for this trace.
|
|
@@ -32605,6 +32666,34 @@ var defaultReporter = {
|
|
|
32605
32666
|
}
|
|
32606
32667
|
};
|
|
32607
32668
|
|
|
32669
|
+
// src/dataset-pipeline.ts
|
|
32670
|
+
function DatasetPipeline(definition) {
|
|
32671
|
+
if (!globalThis.__braintrust_dataset_pipelines) {
|
|
32672
|
+
globalThis.__braintrust_dataset_pipelines = [];
|
|
32673
|
+
}
|
|
32674
|
+
const storedDefinition = {
|
|
32675
|
+
name: definition.name,
|
|
32676
|
+
source: {
|
|
32677
|
+
projectId: definition.source.projectId,
|
|
32678
|
+
projectName: definition.source.projectName,
|
|
32679
|
+
orgName: definition.source.orgName,
|
|
32680
|
+
filter: definition.source.filter,
|
|
32681
|
+
scope: definition.source.scope ?? "span"
|
|
32682
|
+
},
|
|
32683
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions, @typescript-eslint/no-explicit-any
|
|
32684
|
+
transform: definition.transform,
|
|
32685
|
+
target: {
|
|
32686
|
+
projectId: definition.target.projectId,
|
|
32687
|
+
projectName: definition.target.projectName,
|
|
32688
|
+
orgName: definition.target.orgName,
|
|
32689
|
+
datasetName: definition.target.datasetName,
|
|
32690
|
+
description: definition.target.description,
|
|
32691
|
+
metadata: definition.target.metadata
|
|
32692
|
+
}
|
|
32693
|
+
};
|
|
32694
|
+
globalThis.__braintrust_dataset_pipelines.push(storedDefinition);
|
|
32695
|
+
}
|
|
32696
|
+
|
|
32608
32697
|
// src/framework2.ts
|
|
32609
32698
|
import { z as z12 } from "zod/v3";
|
|
32610
32699
|
var currentFilename = typeof __filename !== "undefined" ? __filename : "unknown";
|
|
@@ -33144,6 +33233,7 @@ export {
|
|
|
33144
33233
|
DEFAULT_FETCH_BATCH_SIZE,
|
|
33145
33234
|
DEFAULT_MAX_REQUEST_SIZE,
|
|
33146
33235
|
Dataset2 as Dataset,
|
|
33236
|
+
DatasetPipeline,
|
|
33147
33237
|
ERR_PERMALINK,
|
|
33148
33238
|
Eval,
|
|
33149
33239
|
EvalResultWithSummary,
|