braintrust 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dev/dist/index.d.mts +16 -3
- package/dev/dist/index.d.ts +16 -3
- package/dev/dist/index.js +69 -44
- package/dev/dist/index.mjs +43 -18
- package/dist/browser.d.mts +29 -11
- package/dist/browser.d.ts +29 -11
- package/dist/browser.js +338 -108
- package/dist/browser.mjs +258 -28
- package/dist/cli.js +49 -20
- package/dist/index.d.mts +29 -11
- package/dist/index.d.ts +29 -11
- package/dist/index.js +356 -121
- package/dist/index.mjs +264 -29
- package/package.json +2 -1
package/dev/dist/index.d.mts
CHANGED
|
@@ -8517,11 +8517,24 @@ declare class ObjectFetcher<RecordType> implements AsyncIterable<WithTransaction
|
|
|
8517
8517
|
get id(): Promise<string>;
|
|
8518
8518
|
protected getState(): Promise<BraintrustState>;
|
|
8519
8519
|
private fetchRecordsFromApi;
|
|
8520
|
-
|
|
8520
|
+
/**
|
|
8521
|
+
* Fetch all records from the object.
|
|
8522
|
+
*
|
|
8523
|
+
* @param options Optional parameters for fetching.
|
|
8524
|
+
* @param options.batchSize The number of records to fetch per request. Defaults to 1000.
|
|
8525
|
+
* @returns An async generator of records.
|
|
8526
|
+
*/
|
|
8527
|
+
fetch(options?: {
|
|
8528
|
+
batchSize?: number;
|
|
8529
|
+
}): AsyncGenerator<WithTransactionId<RecordType>>;
|
|
8521
8530
|
[Symbol.asyncIterator](): AsyncIterator<WithTransactionId<RecordType>>;
|
|
8522
|
-
fetchedData(
|
|
8531
|
+
fetchedData(options?: {
|
|
8532
|
+
batchSize?: number;
|
|
8533
|
+
}): Promise<WithTransactionId<RecordType>[]>;
|
|
8523
8534
|
clearCache(): void;
|
|
8524
|
-
version(
|
|
8535
|
+
version(options?: {
|
|
8536
|
+
batchSize?: number;
|
|
8537
|
+
}): Promise<string | undefined>;
|
|
8525
8538
|
}
|
|
8526
8539
|
type BaseMetadata = Record<string, unknown> | void;
|
|
8527
8540
|
type DefaultMetadataType = void;
|
package/dev/dist/index.d.ts
CHANGED
|
@@ -8517,11 +8517,24 @@ declare class ObjectFetcher<RecordType> implements AsyncIterable<WithTransaction
|
|
|
8517
8517
|
get id(): Promise<string>;
|
|
8518
8518
|
protected getState(): Promise<BraintrustState>;
|
|
8519
8519
|
private fetchRecordsFromApi;
|
|
8520
|
-
|
|
8520
|
+
/**
|
|
8521
|
+
* Fetch all records from the object.
|
|
8522
|
+
*
|
|
8523
|
+
* @param options Optional parameters for fetching.
|
|
8524
|
+
* @param options.batchSize The number of records to fetch per request. Defaults to 1000.
|
|
8525
|
+
* @returns An async generator of records.
|
|
8526
|
+
*/
|
|
8527
|
+
fetch(options?: {
|
|
8528
|
+
batchSize?: number;
|
|
8529
|
+
}): AsyncGenerator<WithTransactionId<RecordType>>;
|
|
8521
8530
|
[Symbol.asyncIterator](): AsyncIterator<WithTransactionId<RecordType>>;
|
|
8522
|
-
fetchedData(
|
|
8531
|
+
fetchedData(options?: {
|
|
8532
|
+
batchSize?: number;
|
|
8533
|
+
}): Promise<WithTransactionId<RecordType>[]>;
|
|
8523
8534
|
clearCache(): void;
|
|
8524
|
-
version(
|
|
8535
|
+
version(options?: {
|
|
8536
|
+
batchSize?: number;
|
|
8537
|
+
}): Promise<string | undefined>;
|
|
8525
8538
|
}
|
|
8526
8539
|
type BaseMetadata = Record<string, unknown> | void;
|
|
8527
8540
|
type DefaultMetadataType = void;
|
package/dev/dist/index.js
CHANGED
|
@@ -5852,7 +5852,7 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
|
|
|
5852
5852
|
}
|
|
5853
5853
|
return event;
|
|
5854
5854
|
}
|
|
5855
|
-
var
|
|
5855
|
+
var DEFAULT_FETCH_BATCH_SIZE = 1e3;
|
|
5856
5856
|
var MAX_BTQL_ITERATIONS = 1e4;
|
|
5857
5857
|
var ObjectFetcher = (_class8 = class {
|
|
5858
5858
|
constructor(objectType, pinnedVersion, mutateRecord, _internal_btql) {;_class8.prototype.__init38.call(this);
|
|
@@ -5868,9 +5868,10 @@ var ObjectFetcher = (_class8 = class {
|
|
|
5868
5868
|
async getState() {
|
|
5869
5869
|
throw new Error("ObjectFetcher subclasses must have a 'getState' method");
|
|
5870
5870
|
}
|
|
5871
|
-
async *fetchRecordsFromApi() {
|
|
5871
|
+
async *fetchRecordsFromApi(batchSize) {
|
|
5872
5872
|
const state = await this.getState();
|
|
5873
5873
|
const objectId = await this.id;
|
|
5874
|
+
const limit = _nullishCoalesce(batchSize, () => ( DEFAULT_FETCH_BATCH_SIZE));
|
|
5874
5875
|
let cursor = void 0;
|
|
5875
5876
|
let iterations = 0;
|
|
5876
5877
|
while (true) {
|
|
@@ -5898,7 +5899,7 @@ var ObjectFetcher = (_class8 = class {
|
|
|
5898
5899
|
]
|
|
5899
5900
|
},
|
|
5900
5901
|
cursor,
|
|
5901
|
-
limit
|
|
5902
|
+
limit
|
|
5902
5903
|
},
|
|
5903
5904
|
use_columnstore: false,
|
|
5904
5905
|
brainstore_realtime: true,
|
|
@@ -5924,24 +5925,31 @@ var ObjectFetcher = (_class8 = class {
|
|
|
5924
5925
|
}
|
|
5925
5926
|
}
|
|
5926
5927
|
}
|
|
5927
|
-
|
|
5928
|
+
/**
|
|
5929
|
+
* Fetch all records from the object.
|
|
5930
|
+
*
|
|
5931
|
+
* @param options Optional parameters for fetching.
|
|
5932
|
+
* @param options.batchSize The number of records to fetch per request. Defaults to 1000.
|
|
5933
|
+
* @returns An async generator of records.
|
|
5934
|
+
*/
|
|
5935
|
+
async *fetch(options) {
|
|
5928
5936
|
if (this._fetchedData !== void 0) {
|
|
5929
5937
|
for (const record of this._fetchedData) {
|
|
5930
5938
|
yield record;
|
|
5931
5939
|
}
|
|
5932
5940
|
return;
|
|
5933
5941
|
}
|
|
5934
|
-
for await (const record of this.fetchRecordsFromApi()) {
|
|
5942
|
+
for await (const record of this.fetchRecordsFromApi(_optionalChain([options, 'optionalAccess', _59 => _59.batchSize]))) {
|
|
5935
5943
|
yield record;
|
|
5936
5944
|
}
|
|
5937
5945
|
}
|
|
5938
5946
|
[Symbol.asyncIterator]() {
|
|
5939
5947
|
return this.fetch();
|
|
5940
5948
|
}
|
|
5941
|
-
async fetchedData() {
|
|
5949
|
+
async fetchedData(options) {
|
|
5942
5950
|
if (this._fetchedData === void 0) {
|
|
5943
5951
|
const data = [];
|
|
5944
|
-
for await (const record of this.fetchRecordsFromApi()) {
|
|
5952
|
+
for await (const record of this.fetchRecordsFromApi(_optionalChain([options, 'optionalAccess', _60 => _60.batchSize]))) {
|
|
5945
5953
|
data.push(record);
|
|
5946
5954
|
}
|
|
5947
5955
|
this._fetchedData = data;
|
|
@@ -5951,12 +5959,12 @@ var ObjectFetcher = (_class8 = class {
|
|
|
5951
5959
|
clearCache() {
|
|
5952
5960
|
this._fetchedData = void 0;
|
|
5953
5961
|
}
|
|
5954
|
-
async version() {
|
|
5962
|
+
async version(options) {
|
|
5955
5963
|
if (this.pinnedVersion !== void 0) {
|
|
5956
5964
|
return this.pinnedVersion;
|
|
5957
5965
|
} else {
|
|
5958
5966
|
let maxVersion = void 0;
|
|
5959
|
-
for await (const record of this.fetch()) {
|
|
5967
|
+
for await (const record of this.fetch(options)) {
|
|
5960
5968
|
const xactId = String(_nullishCoalesce(record[TRANSACTION_ID_FIELD], () => ( "0")));
|
|
5961
5969
|
if (maxVersion === void 0 || xactId > maxVersion) {
|
|
5962
5970
|
maxVersion = xactId;
|
|
@@ -6036,7 +6044,7 @@ var Experiment2 = (_class9 = class extends ObjectFetcher {
|
|
|
6036
6044
|
* @returns The `id` of the logged event.
|
|
6037
6045
|
*/
|
|
6038
6046
|
log(event, options) {
|
|
6039
|
-
if (this.calledStartSpan && !_optionalChain([options, 'optionalAccess',
|
|
6047
|
+
if (this.calledStartSpan && !_optionalChain([options, 'optionalAccess', _61 => _61.allowConcurrentWithSpans])) {
|
|
6040
6048
|
throw new Error(
|
|
6041
6049
|
"Cannot run toplevel `log` method while using spans. To log to the span, call `experiment.traced` and then log with `span.log`"
|
|
6042
6050
|
);
|
|
@@ -6089,12 +6097,12 @@ var Experiment2 = (_class9 = class extends ObjectFetcher {
|
|
|
6089
6097
|
state: this.state,
|
|
6090
6098
|
...startSpanParentArgs({
|
|
6091
6099
|
state: this.state,
|
|
6092
|
-
parent: _optionalChain([args, 'optionalAccess',
|
|
6100
|
+
parent: _optionalChain([args, 'optionalAccess', _62 => _62.parent]),
|
|
6093
6101
|
parentObjectType: this.parentObjectType(),
|
|
6094
6102
|
parentObjectId: this.lazyId,
|
|
6095
6103
|
parentComputeObjectMetadataArgs: void 0,
|
|
6096
6104
|
parentSpanIds: void 0,
|
|
6097
|
-
propagatedEvent: _optionalChain([args, 'optionalAccess',
|
|
6105
|
+
propagatedEvent: _optionalChain([args, 'optionalAccess', _63 => _63.propagatedEvent])
|
|
6098
6106
|
}),
|
|
6099
6107
|
defaultRootType: "eval" /* EVAL */
|
|
6100
6108
|
});
|
|
@@ -6264,8 +6272,8 @@ var ReadonlyExperiment = class extends ObjectFetcher {
|
|
|
6264
6272
|
await this.lazyMetadata.get();
|
|
6265
6273
|
return this.state;
|
|
6266
6274
|
}
|
|
6267
|
-
async *asDataset() {
|
|
6268
|
-
const records = this.fetch();
|
|
6275
|
+
async *asDataset(options) {
|
|
6276
|
+
const records = this.fetch(options);
|
|
6269
6277
|
for await (const record of records) {
|
|
6270
6278
|
if (record.root_span_id !== record.span_id) {
|
|
6271
6279
|
continue;
|
|
@@ -6432,10 +6440,10 @@ var SpanImpl = (_class10 = class _SpanImpl {
|
|
|
6432
6440
|
...serializableInternalData,
|
|
6433
6441
|
[IS_MERGE_FIELD]: this.isMerge
|
|
6434
6442
|
});
|
|
6435
|
-
if (_optionalChain([partialRecord, 'access',
|
|
6436
|
-
this.loggedEndTime = _optionalChain([partialRecord, 'access',
|
|
6443
|
+
if (_optionalChain([partialRecord, 'access', _64 => _64.metrics, 'optionalAccess', _65 => _65.end])) {
|
|
6444
|
+
this.loggedEndTime = _optionalChain([partialRecord, 'access', _66 => _66.metrics, 'optionalAccess', _67 => _67.end]);
|
|
6437
6445
|
}
|
|
6438
|
-
if ((_nullishCoalesce(partialRecord.tags, () => ( []))).length > 0 && _optionalChain([this, 'access',
|
|
6446
|
+
if ((_nullishCoalesce(partialRecord.tags, () => ( []))).length > 0 && _optionalChain([this, 'access', _68 => _68._spanParents, 'optionalAccess', _69 => _69.length])) {
|
|
6439
6447
|
throw new Error("Tags can only be logged to the root span");
|
|
6440
6448
|
}
|
|
6441
6449
|
const computeRecord = async () => ({
|
|
@@ -6480,18 +6488,18 @@ var SpanImpl = (_class10 = class _SpanImpl {
|
|
|
6480
6488
|
);
|
|
6481
6489
|
}
|
|
6482
6490
|
startSpan(args) {
|
|
6483
|
-
const parentSpanIds = _optionalChain([args, 'optionalAccess',
|
|
6491
|
+
const parentSpanIds = _optionalChain([args, 'optionalAccess', _70 => _70.parent]) ? void 0 : { spanId: this._spanId, rootSpanId: this._rootSpanId };
|
|
6484
6492
|
return new _SpanImpl({
|
|
6485
6493
|
state: this._state,
|
|
6486
6494
|
...args,
|
|
6487
6495
|
...startSpanParentArgs({
|
|
6488
6496
|
state: this._state,
|
|
6489
|
-
parent: _optionalChain([args, 'optionalAccess',
|
|
6497
|
+
parent: _optionalChain([args, 'optionalAccess', _71 => _71.parent]),
|
|
6490
6498
|
parentObjectType: this.parentObjectType,
|
|
6491
6499
|
parentObjectId: this.parentObjectId,
|
|
6492
6500
|
parentComputeObjectMetadataArgs: this.parentComputeObjectMetadataArgs,
|
|
6493
6501
|
parentSpanIds,
|
|
6494
|
-
propagatedEvent: _nullishCoalesce(_optionalChain([args, 'optionalAccess',
|
|
6502
|
+
propagatedEvent: _nullishCoalesce(_optionalChain([args, 'optionalAccess', _72 => _72.propagatedEvent]), () => ( this.propagatedEvent))
|
|
6495
6503
|
})
|
|
6496
6504
|
});
|
|
6497
6505
|
}
|
|
@@ -6505,12 +6513,12 @@ var SpanImpl = (_class10 = class _SpanImpl {
|
|
|
6505
6513
|
...args,
|
|
6506
6514
|
...startSpanParentArgs({
|
|
6507
6515
|
state: this._state,
|
|
6508
|
-
parent: _optionalChain([args, 'optionalAccess',
|
|
6516
|
+
parent: _optionalChain([args, 'optionalAccess', _73 => _73.parent]),
|
|
6509
6517
|
parentObjectType: this.parentObjectType,
|
|
6510
6518
|
parentObjectId: this.parentObjectId,
|
|
6511
6519
|
parentComputeObjectMetadataArgs: this.parentComputeObjectMetadataArgs,
|
|
6512
6520
|
parentSpanIds,
|
|
6513
|
-
propagatedEvent: _nullishCoalesce(_optionalChain([args, 'optionalAccess',
|
|
6521
|
+
propagatedEvent: _nullishCoalesce(_optionalChain([args, 'optionalAccess', _74 => _74.propagatedEvent]), () => ( this.propagatedEvent))
|
|
6514
6522
|
}),
|
|
6515
6523
|
spanId
|
|
6516
6524
|
});
|
|
@@ -6519,7 +6527,7 @@ var SpanImpl = (_class10 = class _SpanImpl {
|
|
|
6519
6527
|
let endTime;
|
|
6520
6528
|
let internalData = {};
|
|
6521
6529
|
if (!this.loggedEndTime) {
|
|
6522
|
-
endTime = _nullishCoalesce(_optionalChain([args, 'optionalAccess',
|
|
6530
|
+
endTime = _nullishCoalesce(_optionalChain([args, 'optionalAccess', _75 => _75.endTime]), () => ( getCurrentUnixTimestamp()));
|
|
6523
6531
|
internalData = { metrics: { end: endTime } };
|
|
6524
6532
|
} else {
|
|
6525
6533
|
endTime = this.loggedEndTime;
|
|
@@ -6562,8 +6570,8 @@ var SpanImpl = (_class10 = class _SpanImpl {
|
|
|
6562
6570
|
const args = this.parentComputeObjectMetadataArgs;
|
|
6563
6571
|
switch (this.parentObjectType) {
|
|
6564
6572
|
case 2 /* PROJECT_LOGS */: {
|
|
6565
|
-
const projectID = _optionalChain([args, 'optionalAccess',
|
|
6566
|
-
const projectName = _optionalChain([args, 'optionalAccess',
|
|
6573
|
+
const projectID = _optionalChain([args, 'optionalAccess', _76 => _76.project_id]) || this.parentObjectId.getSync().value;
|
|
6574
|
+
const projectName = _optionalChain([args, 'optionalAccess', _77 => _77.project_name]);
|
|
6567
6575
|
if (projectID) {
|
|
6568
6576
|
return `${baseUrl}/object?object_type=project_logs&object_id=${projectID}&id=${this._id}`;
|
|
6569
6577
|
} else if (projectName) {
|
|
@@ -6573,7 +6581,7 @@ var SpanImpl = (_class10 = class _SpanImpl {
|
|
|
6573
6581
|
}
|
|
6574
6582
|
}
|
|
6575
6583
|
case 1 /* EXPERIMENT */: {
|
|
6576
|
-
const expID = _optionalChain([args, 'optionalAccess',
|
|
6584
|
+
const expID = _optionalChain([args, 'optionalAccess', _78 => _78.experiment_id]) || _optionalChain([this, 'access', _79 => _79.parentObjectId, 'optionalAccess', _80 => _80.getSync, 'call', _81 => _81(), 'optionalAccess', _82 => _82.value]);
|
|
6577
6585
|
if (!expID) {
|
|
6578
6586
|
return getErrPermlink("provide-experiment-id");
|
|
6579
6587
|
} else {
|
|
@@ -7029,13 +7037,13 @@ var Prompt2 = (_class12 = class _Prompt {
|
|
|
7029
7037
|
return "slug" in this.metadata ? this.metadata.slug : this.metadata.id;
|
|
7030
7038
|
}
|
|
7031
7039
|
get prompt() {
|
|
7032
|
-
return _optionalChain([this, 'access',
|
|
7040
|
+
return _optionalChain([this, 'access', _83 => _83.getParsedPromptData, 'call', _84 => _84(), 'optionalAccess', _85 => _85.prompt]);
|
|
7033
7041
|
}
|
|
7034
7042
|
get version() {
|
|
7035
7043
|
return this.metadata[TRANSACTION_ID_FIELD];
|
|
7036
7044
|
}
|
|
7037
7045
|
get options() {
|
|
7038
|
-
return _optionalChain([this, 'access',
|
|
7046
|
+
return _optionalChain([this, 'access', _86 => _86.getParsedPromptData, 'call', _87 => _87(), 'optionalAccess', _88 => _88.options]) || {};
|
|
7039
7047
|
}
|
|
7040
7048
|
get promptData() {
|
|
7041
7049
|
return this.getParsedPromptData();
|
|
@@ -7186,7 +7194,7 @@ var Prompt2 = (_class12 = class _Prompt {
|
|
|
7186
7194
|
return {
|
|
7187
7195
|
type: "chat",
|
|
7188
7196
|
messages,
|
|
7189
|
-
..._optionalChain([prompt, 'access',
|
|
7197
|
+
..._optionalChain([prompt, 'access', _89 => _89.tools, 'optionalAccess', _90 => _90.trim, 'call', _91 => _91()]) ? {
|
|
7190
7198
|
tools: render(prompt.tools)
|
|
7191
7199
|
} : void 0
|
|
7192
7200
|
};
|
|
@@ -8305,6 +8313,7 @@ var waterfall$1 = awaitify(waterfall);
|
|
|
8305
8313
|
|
|
8306
8314
|
// src/framework.ts
|
|
8307
8315
|
var _chalk = require('chalk'); var _chalk2 = _interopRequireDefault(_chalk);
|
|
8316
|
+
var _termilink = require('termi-link');
|
|
8308
8317
|
var _boxen = require('boxen'); var _boxen2 = _interopRequireDefault(_boxen);
|
|
8309
8318
|
var _pluralize = require('pluralize'); var _pluralize2 = _interopRequireDefault(_pluralize);
|
|
8310
8319
|
var _clitable3 = require('cli-table3'); var _clitable32 = _interopRequireDefault(_clitable3);
|
|
@@ -9055,10 +9064,10 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
9055
9064
|
span,
|
|
9056
9065
|
parameters: _nullishCoalesce(parameters, () => ( {})),
|
|
9057
9066
|
reportProgress: (event) => {
|
|
9058
|
-
_optionalChain([stream, 'optionalCall',
|
|
9067
|
+
_optionalChain([stream, 'optionalCall', _92 => _92({
|
|
9059
9068
|
...event,
|
|
9060
9069
|
id: rootSpan.id,
|
|
9061
|
-
origin: _optionalChain([baseEvent, 'access',
|
|
9070
|
+
origin: _optionalChain([baseEvent, 'access', _93 => _93.event, 'optionalAccess', _94 => _94.origin]),
|
|
9062
9071
|
name: evaluator.evalName,
|
|
9063
9072
|
object_type: "task"
|
|
9064
9073
|
})]);
|
|
@@ -9213,7 +9222,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
9213
9222
|
metadata,
|
|
9214
9223
|
scores: mergedScores,
|
|
9215
9224
|
error: error2,
|
|
9216
|
-
origin: _optionalChain([baseEvent, 'access',
|
|
9225
|
+
origin: _optionalChain([baseEvent, 'access', _95 => _95.event, 'optionalAccess', _96 => _96.origin])
|
|
9217
9226
|
});
|
|
9218
9227
|
}
|
|
9219
9228
|
};
|
|
@@ -9246,7 +9255,7 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
9246
9255
|
break;
|
|
9247
9256
|
}
|
|
9248
9257
|
scheduledTrials++;
|
|
9249
|
-
_optionalChain([progressReporter, 'access',
|
|
9258
|
+
_optionalChain([progressReporter, 'access', _97 => _97.setTotal, 'optionalCall', _98 => _98(evaluator.evalName, scheduledTrials)]);
|
|
9250
9259
|
q.push({ datum, trialIndex });
|
|
9251
9260
|
}
|
|
9252
9261
|
}
|
|
@@ -9508,7 +9517,11 @@ function formatExperimentSummary(summary) {
|
|
|
9508
9517
|
tableParts.push(combinedTable.toString());
|
|
9509
9518
|
}
|
|
9510
9519
|
const content = [comparisonLine, ...tableParts].filter(Boolean).join("\n");
|
|
9511
|
-
const footer = summary.experimentUrl ?
|
|
9520
|
+
const footer = summary.experimentUrl ? _termilink.terminalLink.call(void 0,
|
|
9521
|
+
`View results for ${summary.experimentName}`,
|
|
9522
|
+
summary.experimentUrl,
|
|
9523
|
+
{ fallback: () => `See results at ${summary.experimentUrl}` }
|
|
9524
|
+
) : "";
|
|
9512
9525
|
const boxContent = [content, footer].filter(Boolean).join("\n\n");
|
|
9513
9526
|
return "\n" + _boxen2.default.call(void 0, boxContent, {
|
|
9514
9527
|
title: _chalk2.default.gray("Experiment summary"),
|
|
@@ -9556,7 +9569,8 @@ function authorizeRequest(req, res, next) {
|
|
|
9556
9569
|
const ctx = {
|
|
9557
9570
|
appOrigin: extractAllowedOrigin(req.headers[ORIGIN_HEADER]),
|
|
9558
9571
|
token: void 0,
|
|
9559
|
-
state: void 0
|
|
9572
|
+
state: void 0,
|
|
9573
|
+
projectId: parseHeader(req.headers, PROJECT_ID_HEADER)
|
|
9560
9574
|
};
|
|
9561
9575
|
if (req.headers.authorization || req.headers[BRAINTRUST_AUTH_TOKEN_HEADER]) {
|
|
9562
9576
|
const tokenText = parseBraintrustAuthHeader(req.headers);
|
|
@@ -9587,7 +9601,7 @@ async function cachedLogin(options) {
|
|
|
9587
9601
|
}
|
|
9588
9602
|
function makeCheckAuthorized(allowedOrgName) {
|
|
9589
9603
|
return async (req, _res, next) => {
|
|
9590
|
-
if (!_optionalChain([req, 'access',
|
|
9604
|
+
if (!_optionalChain([req, 'access', _99 => _99.ctx, 'optionalAccess', _100 => _100.token])) {
|
|
9591
9605
|
return next(_httperrors2.default.call(void 0, 401, "Unauthorized"));
|
|
9592
9606
|
}
|
|
9593
9607
|
try {
|
|
@@ -9600,7 +9614,7 @@ function makeCheckAuthorized(allowedOrgName) {
|
|
|
9600
9614
|
return next(_httperrors2.default.call(void 0, 403, errorMessage));
|
|
9601
9615
|
}
|
|
9602
9616
|
const state = await cachedLogin({
|
|
9603
|
-
apiKey: _optionalChain([req, 'access',
|
|
9617
|
+
apiKey: _optionalChain([req, 'access', _101 => _101.ctx, 'optionalAccess', _102 => _102.token]),
|
|
9604
9618
|
orgName
|
|
9605
9619
|
});
|
|
9606
9620
|
req.ctx.state = state;
|
|
@@ -9638,6 +9652,7 @@ function checkOrigin(requestOrigin, callback) {
|
|
|
9638
9652
|
}
|
|
9639
9653
|
var BRAINTRUST_AUTH_TOKEN_HEADER = "x-bt-auth-token";
|
|
9640
9654
|
var ORIGIN_HEADER = "origin";
|
|
9655
|
+
var PROJECT_ID_HEADER = "x-bt-project-id";
|
|
9641
9656
|
function extractAllowedOrigin(originHeader) {
|
|
9642
9657
|
let allowedOrigin = MAIN_ORIGIN;
|
|
9643
9658
|
checkOrigin(originHeader, (err, origin) => {
|
|
@@ -9684,6 +9699,7 @@ var baseAllowedHeaders = [
|
|
|
9684
9699
|
"x-bt-parent",
|
|
9685
9700
|
// These are eval-specific
|
|
9686
9701
|
"x-bt-org-name",
|
|
9702
|
+
"x-bt-project-id",
|
|
9687
9703
|
"x-bt-stream-fmt",
|
|
9688
9704
|
"x-bt-use-cache",
|
|
9689
9705
|
"x-stainless-os",
|
|
@@ -9806,7 +9822,7 @@ function runDevServer(evaluators, opts) {
|
|
|
9806
9822
|
scores,
|
|
9807
9823
|
stream
|
|
9808
9824
|
} = evalBodySchema.parse(req.body);
|
|
9809
|
-
if (!_optionalChain([req, 'access',
|
|
9825
|
+
if (!_optionalChain([req, 'access', _103 => _103.ctx, 'optionalAccess', _104 => _104.state])) {
|
|
9810
9826
|
res.status(500).json({ error: "Braintrust state not initialized in request" });
|
|
9811
9827
|
return;
|
|
9812
9828
|
}
|
|
@@ -9863,8 +9879,13 @@ function runDevServer(evaluators, opts) {
|
|
|
9863
9879
|
...evaluator,
|
|
9864
9880
|
data: evalData.data,
|
|
9865
9881
|
scores: evaluator.scores.concat(
|
|
9866
|
-
_nullishCoalesce(_optionalChain([scores, 'optionalAccess',
|
|
9867
|
-
(score) => makeScorer(
|
|
9882
|
+
_nullishCoalesce(_optionalChain([scores, 'optionalAccess', _105 => _105.map, 'call', _106 => _106(
|
|
9883
|
+
(score) => makeScorer(
|
|
9884
|
+
state,
|
|
9885
|
+
score.name,
|
|
9886
|
+
score.function_id,
|
|
9887
|
+
_optionalChain([req, 'access', _107 => _107.ctx, 'optionalAccess', _108 => _108.projectId])
|
|
9888
|
+
)
|
|
9868
9889
|
)]), () => ( []))
|
|
9869
9890
|
),
|
|
9870
9891
|
task,
|
|
@@ -9988,7 +10009,7 @@ async function getDatasetById({
|
|
|
9988
10009
|
}
|
|
9989
10010
|
return { projectId: parsed[0].project_id, dataset: parsed[0].name };
|
|
9990
10011
|
}
|
|
9991
|
-
function makeScorer(state, name, score) {
|
|
10012
|
+
function makeScorer(state, name, score, projectId) {
|
|
9992
10013
|
const ret = async (input) => {
|
|
9993
10014
|
const request = {
|
|
9994
10015
|
...score,
|
|
@@ -9998,10 +10019,14 @@ function makeScorer(state, name, score) {
|
|
|
9998
10019
|
mode: "auto",
|
|
9999
10020
|
strict: true
|
|
10000
10021
|
};
|
|
10022
|
+
const headers = {
|
|
10023
|
+
Accept: "application/json"
|
|
10024
|
+
};
|
|
10025
|
+
if (projectId) {
|
|
10026
|
+
headers["x-bt-project-id"] = projectId;
|
|
10027
|
+
}
|
|
10001
10028
|
const result = await state.proxyConn().post(`function/invoke`, request, {
|
|
10002
|
-
headers
|
|
10003
|
-
Accept: "application/json"
|
|
10004
|
-
}
|
|
10029
|
+
headers
|
|
10005
10030
|
});
|
|
10006
10031
|
const data = await result.json();
|
|
10007
10032
|
return data;
|
package/dev/dist/index.mjs
CHANGED
|
@@ -5852,7 +5852,7 @@ function validateAndSanitizeExperimentLogFullArgs(event, hasDataset) {
|
|
|
5852
5852
|
}
|
|
5853
5853
|
return event;
|
|
5854
5854
|
}
|
|
5855
|
-
var
|
|
5855
|
+
var DEFAULT_FETCH_BATCH_SIZE = 1e3;
|
|
5856
5856
|
var MAX_BTQL_ITERATIONS = 1e4;
|
|
5857
5857
|
var ObjectFetcher = class {
|
|
5858
5858
|
constructor(objectType, pinnedVersion, mutateRecord, _internal_btql) {
|
|
@@ -5868,9 +5868,10 @@ var ObjectFetcher = class {
|
|
|
5868
5868
|
async getState() {
|
|
5869
5869
|
throw new Error("ObjectFetcher subclasses must have a 'getState' method");
|
|
5870
5870
|
}
|
|
5871
|
-
async *fetchRecordsFromApi() {
|
|
5871
|
+
async *fetchRecordsFromApi(batchSize) {
|
|
5872
5872
|
const state = await this.getState();
|
|
5873
5873
|
const objectId = await this.id;
|
|
5874
|
+
const limit = batchSize ?? DEFAULT_FETCH_BATCH_SIZE;
|
|
5874
5875
|
let cursor = void 0;
|
|
5875
5876
|
let iterations = 0;
|
|
5876
5877
|
while (true) {
|
|
@@ -5898,7 +5899,7 @@ var ObjectFetcher = class {
|
|
|
5898
5899
|
]
|
|
5899
5900
|
},
|
|
5900
5901
|
cursor,
|
|
5901
|
-
limit
|
|
5902
|
+
limit
|
|
5902
5903
|
},
|
|
5903
5904
|
use_columnstore: false,
|
|
5904
5905
|
brainstore_realtime: true,
|
|
@@ -5924,24 +5925,31 @@ var ObjectFetcher = class {
|
|
|
5924
5925
|
}
|
|
5925
5926
|
}
|
|
5926
5927
|
}
|
|
5927
|
-
|
|
5928
|
+
/**
|
|
5929
|
+
* Fetch all records from the object.
|
|
5930
|
+
*
|
|
5931
|
+
* @param options Optional parameters for fetching.
|
|
5932
|
+
* @param options.batchSize The number of records to fetch per request. Defaults to 1000.
|
|
5933
|
+
* @returns An async generator of records.
|
|
5934
|
+
*/
|
|
5935
|
+
async *fetch(options) {
|
|
5928
5936
|
if (this._fetchedData !== void 0) {
|
|
5929
5937
|
for (const record of this._fetchedData) {
|
|
5930
5938
|
yield record;
|
|
5931
5939
|
}
|
|
5932
5940
|
return;
|
|
5933
5941
|
}
|
|
5934
|
-
for await (const record of this.fetchRecordsFromApi()) {
|
|
5942
|
+
for await (const record of this.fetchRecordsFromApi(options?.batchSize)) {
|
|
5935
5943
|
yield record;
|
|
5936
5944
|
}
|
|
5937
5945
|
}
|
|
5938
5946
|
[Symbol.asyncIterator]() {
|
|
5939
5947
|
return this.fetch();
|
|
5940
5948
|
}
|
|
5941
|
-
async fetchedData() {
|
|
5949
|
+
async fetchedData(options) {
|
|
5942
5950
|
if (this._fetchedData === void 0) {
|
|
5943
5951
|
const data = [];
|
|
5944
|
-
for await (const record of this.fetchRecordsFromApi()) {
|
|
5952
|
+
for await (const record of this.fetchRecordsFromApi(options?.batchSize)) {
|
|
5945
5953
|
data.push(record);
|
|
5946
5954
|
}
|
|
5947
5955
|
this._fetchedData = data;
|
|
@@ -5951,12 +5959,12 @@ var ObjectFetcher = class {
|
|
|
5951
5959
|
clearCache() {
|
|
5952
5960
|
this._fetchedData = void 0;
|
|
5953
5961
|
}
|
|
5954
|
-
async version() {
|
|
5962
|
+
async version(options) {
|
|
5955
5963
|
if (this.pinnedVersion !== void 0) {
|
|
5956
5964
|
return this.pinnedVersion;
|
|
5957
5965
|
} else {
|
|
5958
5966
|
let maxVersion = void 0;
|
|
5959
|
-
for await (const record of this.fetch()) {
|
|
5967
|
+
for await (const record of this.fetch(options)) {
|
|
5960
5968
|
const xactId = String(record[TRANSACTION_ID_FIELD] ?? "0");
|
|
5961
5969
|
if (maxVersion === void 0 || xactId > maxVersion) {
|
|
5962
5970
|
maxVersion = xactId;
|
|
@@ -6264,8 +6272,8 @@ var ReadonlyExperiment = class extends ObjectFetcher {
|
|
|
6264
6272
|
await this.lazyMetadata.get();
|
|
6265
6273
|
return this.state;
|
|
6266
6274
|
}
|
|
6267
|
-
async *asDataset() {
|
|
6268
|
-
const records = this.fetch();
|
|
6275
|
+
async *asDataset(options) {
|
|
6276
|
+
const records = this.fetch(options);
|
|
6269
6277
|
for await (const record of records) {
|
|
6270
6278
|
if (record.root_span_id !== record.span_id) {
|
|
6271
6279
|
continue;
|
|
@@ -8305,6 +8313,7 @@ var waterfall$1 = awaitify(waterfall);
|
|
|
8305
8313
|
|
|
8306
8314
|
// src/framework.ts
|
|
8307
8315
|
import chalk2 from "chalk";
|
|
8316
|
+
import { terminalLink } from "termi-link";
|
|
8308
8317
|
import boxen from "boxen";
|
|
8309
8318
|
import pluralize from "pluralize";
|
|
8310
8319
|
import Table from "cli-table3";
|
|
@@ -9508,7 +9517,11 @@ function formatExperimentSummary(summary) {
|
|
|
9508
9517
|
tableParts.push(combinedTable.toString());
|
|
9509
9518
|
}
|
|
9510
9519
|
const content = [comparisonLine, ...tableParts].filter(Boolean).join("\n");
|
|
9511
|
-
const footer = summary.experimentUrl ?
|
|
9520
|
+
const footer = summary.experimentUrl ? terminalLink(
|
|
9521
|
+
`View results for ${summary.experimentName}`,
|
|
9522
|
+
summary.experimentUrl,
|
|
9523
|
+
{ fallback: () => `See results at ${summary.experimentUrl}` }
|
|
9524
|
+
) : "";
|
|
9512
9525
|
const boxContent = [content, footer].filter(Boolean).join("\n\n");
|
|
9513
9526
|
return "\n" + boxen(boxContent, {
|
|
9514
9527
|
title: chalk2.gray("Experiment summary"),
|
|
@@ -9556,7 +9569,8 @@ function authorizeRequest(req, res, next) {
|
|
|
9556
9569
|
const ctx = {
|
|
9557
9570
|
appOrigin: extractAllowedOrigin(req.headers[ORIGIN_HEADER]),
|
|
9558
9571
|
token: void 0,
|
|
9559
|
-
state: void 0
|
|
9572
|
+
state: void 0,
|
|
9573
|
+
projectId: parseHeader(req.headers, PROJECT_ID_HEADER)
|
|
9560
9574
|
};
|
|
9561
9575
|
if (req.headers.authorization || req.headers[BRAINTRUST_AUTH_TOKEN_HEADER]) {
|
|
9562
9576
|
const tokenText = parseBraintrustAuthHeader(req.headers);
|
|
@@ -9638,6 +9652,7 @@ function checkOrigin(requestOrigin, callback) {
|
|
|
9638
9652
|
}
|
|
9639
9653
|
var BRAINTRUST_AUTH_TOKEN_HEADER = "x-bt-auth-token";
|
|
9640
9654
|
var ORIGIN_HEADER = "origin";
|
|
9655
|
+
var PROJECT_ID_HEADER = "x-bt-project-id";
|
|
9641
9656
|
function extractAllowedOrigin(originHeader) {
|
|
9642
9657
|
let allowedOrigin = MAIN_ORIGIN;
|
|
9643
9658
|
checkOrigin(originHeader, (err, origin) => {
|
|
@@ -9684,6 +9699,7 @@ var baseAllowedHeaders = [
|
|
|
9684
9699
|
"x-bt-parent",
|
|
9685
9700
|
// These are eval-specific
|
|
9686
9701
|
"x-bt-org-name",
|
|
9702
|
+
"x-bt-project-id",
|
|
9687
9703
|
"x-bt-stream-fmt",
|
|
9688
9704
|
"x-bt-use-cache",
|
|
9689
9705
|
"x-stainless-os",
|
|
@@ -9864,7 +9880,12 @@ function runDevServer(evaluators, opts) {
|
|
|
9864
9880
|
data: evalData.data,
|
|
9865
9881
|
scores: evaluator.scores.concat(
|
|
9866
9882
|
scores?.map(
|
|
9867
|
-
(score) => makeScorer(
|
|
9883
|
+
(score) => makeScorer(
|
|
9884
|
+
state,
|
|
9885
|
+
score.name,
|
|
9886
|
+
score.function_id,
|
|
9887
|
+
req.ctx?.projectId
|
|
9888
|
+
)
|
|
9868
9889
|
) ?? []
|
|
9869
9890
|
),
|
|
9870
9891
|
task,
|
|
@@ -9988,7 +10009,7 @@ async function getDatasetById({
|
|
|
9988
10009
|
}
|
|
9989
10010
|
return { projectId: parsed[0].project_id, dataset: parsed[0].name };
|
|
9990
10011
|
}
|
|
9991
|
-
function makeScorer(state, name, score) {
|
|
10012
|
+
function makeScorer(state, name, score, projectId) {
|
|
9992
10013
|
const ret = async (input) => {
|
|
9993
10014
|
const request = {
|
|
9994
10015
|
...score,
|
|
@@ -9998,10 +10019,14 @@ function makeScorer(state, name, score) {
|
|
|
9998
10019
|
mode: "auto",
|
|
9999
10020
|
strict: true
|
|
10000
10021
|
};
|
|
10022
|
+
const headers = {
|
|
10023
|
+
Accept: "application/json"
|
|
10024
|
+
};
|
|
10025
|
+
if (projectId) {
|
|
10026
|
+
headers["x-bt-project-id"] = projectId;
|
|
10027
|
+
}
|
|
10001
10028
|
const result = await state.proxyConn().post(`function/invoke`, request, {
|
|
10002
|
-
headers
|
|
10003
|
-
Accept: "application/json"
|
|
10004
|
-
}
|
|
10029
|
+
headers
|
|
10005
10030
|
});
|
|
10006
10031
|
const data = await result.json();
|
|
10007
10032
|
return data;
|