braintrust 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dev/dist/index.js +17 -4
- package/dev/dist/index.mjs +17 -4
- package/dist/browser.d.mts +3 -1
- package/dist/browser.d.ts +3 -1
- package/dist/browser.js +20 -5
- package/dist/browser.mjs +20 -5
- package/dist/cli.js +22 -7
- package/dist/index.d.mts +50 -3
- package/dist/index.d.ts +50 -3
- package/dist/index.js +251 -23
- package/dist/index.mjs +236 -8
- package/package.json +5 -3
package/dev/dist/index.js
CHANGED
|
@@ -6677,6 +6677,8 @@ var evalBodySchema = _zod.z.object({
|
|
|
6677
6677
|
name: _zod.z.string()
|
|
6678
6678
|
})
|
|
6679
6679
|
).nullish(),
|
|
6680
|
+
experiment_name: _zod.z.string().nullish(),
|
|
6681
|
+
project_id: _zod.z.string().nullish(),
|
|
6680
6682
|
parent: _typespecs.invokeParent.optional(),
|
|
6681
6683
|
stream: _zod.z.boolean().optional()
|
|
6682
6684
|
});
|
|
@@ -6759,7 +6761,16 @@ function runDevServer(evaluators, opts) {
|
|
|
6759
6761
|
"/eval",
|
|
6760
6762
|
checkAuthorized,
|
|
6761
6763
|
asyncHandler(async (req, res) => {
|
|
6762
|
-
const {
|
|
6764
|
+
const {
|
|
6765
|
+
name,
|
|
6766
|
+
parameters,
|
|
6767
|
+
parent,
|
|
6768
|
+
experiment_name,
|
|
6769
|
+
project_id,
|
|
6770
|
+
data,
|
|
6771
|
+
scores,
|
|
6772
|
+
stream
|
|
6773
|
+
} = evalBodySchema.parse(req.body);
|
|
6763
6774
|
const state = await cachedLogin({ apiKey: _optionalChain([req, 'access', _88 => _88.ctx, 'optionalAccess', _89 => _89.token]) });
|
|
6764
6775
|
const evaluator = allEvaluators[name];
|
|
6765
6776
|
if (!evaluator) {
|
|
@@ -6818,17 +6829,19 @@ function runDevServer(evaluators, opts) {
|
|
|
6818
6829
|
)]), () => ( []))
|
|
6819
6830
|
),
|
|
6820
6831
|
task,
|
|
6821
|
-
state
|
|
6832
|
+
state,
|
|
6833
|
+
experimentName: _nullishCoalesce(experiment_name, () => ( void 0)),
|
|
6834
|
+
projectId: _nullishCoalesce(project_id, () => ( void 0))
|
|
6822
6835
|
},
|
|
6823
6836
|
{
|
|
6824
6837
|
// Avoid printing the bar to the console.
|
|
6825
6838
|
progress: {
|
|
6826
|
-
start: (
|
|
6839
|
+
start: () => {
|
|
6827
6840
|
},
|
|
6828
6841
|
stop: () => {
|
|
6829
6842
|
console.log("Finished running experiment");
|
|
6830
6843
|
},
|
|
6831
|
-
increment: (
|
|
6844
|
+
increment: () => {
|
|
6832
6845
|
}
|
|
6833
6846
|
},
|
|
6834
6847
|
stream: (data2) => {
|
package/dev/dist/index.mjs
CHANGED
|
@@ -6677,6 +6677,8 @@ var evalBodySchema = z6.object({
|
|
|
6677
6677
|
name: z6.string()
|
|
6678
6678
|
})
|
|
6679
6679
|
).nullish(),
|
|
6680
|
+
experiment_name: z6.string().nullish(),
|
|
6681
|
+
project_id: z6.string().nullish(),
|
|
6680
6682
|
parent: invokeParent.optional(),
|
|
6681
6683
|
stream: z6.boolean().optional()
|
|
6682
6684
|
});
|
|
@@ -6759,7 +6761,16 @@ function runDevServer(evaluators, opts) {
|
|
|
6759
6761
|
"/eval",
|
|
6760
6762
|
checkAuthorized,
|
|
6761
6763
|
asyncHandler(async (req, res) => {
|
|
6762
|
-
const {
|
|
6764
|
+
const {
|
|
6765
|
+
name,
|
|
6766
|
+
parameters,
|
|
6767
|
+
parent,
|
|
6768
|
+
experiment_name,
|
|
6769
|
+
project_id,
|
|
6770
|
+
data,
|
|
6771
|
+
scores,
|
|
6772
|
+
stream
|
|
6773
|
+
} = evalBodySchema.parse(req.body);
|
|
6763
6774
|
const state = await cachedLogin({ apiKey: req.ctx?.token });
|
|
6764
6775
|
const evaluator = allEvaluators[name];
|
|
6765
6776
|
if (!evaluator) {
|
|
@@ -6818,17 +6829,19 @@ function runDevServer(evaluators, opts) {
|
|
|
6818
6829
|
) ?? []
|
|
6819
6830
|
),
|
|
6820
6831
|
task,
|
|
6821
|
-
state
|
|
6832
|
+
state,
|
|
6833
|
+
experimentName: experiment_name ?? void 0,
|
|
6834
|
+
projectId: project_id ?? void 0
|
|
6822
6835
|
},
|
|
6823
6836
|
{
|
|
6824
6837
|
// Avoid printing the bar to the console.
|
|
6825
6838
|
progress: {
|
|
6826
|
-
start: (
|
|
6839
|
+
start: () => {
|
|
6827
6840
|
},
|
|
6828
6841
|
stop: () => {
|
|
6829
6842
|
console.log("Finished running experiment");
|
|
6830
6843
|
},
|
|
6831
|
-
increment: (
|
|
6844
|
+
increment: () => {
|
|
6832
6845
|
}
|
|
6833
6846
|
},
|
|
6834
6847
|
stream: (data2) => {
|
package/dist/browser.d.mts
CHANGED
|
@@ -1690,6 +1690,7 @@ interface DatasetSummary {
|
|
|
1690
1690
|
datasetUrl: string;
|
|
1691
1691
|
dataSummary: DataSummary | undefined;
|
|
1692
1692
|
}
|
|
1693
|
+
declare function setInitialTestState(): void;
|
|
1693
1694
|
declare function simulateLoginForTests(): Promise<BraintrustState>;
|
|
1694
1695
|
declare function simulateLogoutForTests(): BraintrustState;
|
|
1695
1696
|
declare const _exportsForTestingOnly: {
|
|
@@ -1699,6 +1700,7 @@ declare const _exportsForTestingOnly: {
|
|
|
1699
1700
|
clearTestBackgroundLogger: typeof clearTestBackgroundLogger;
|
|
1700
1701
|
simulateLoginForTests: typeof simulateLoginForTests;
|
|
1701
1702
|
simulateLogoutForTests: typeof simulateLogoutForTests;
|
|
1703
|
+
setInitialTestState: typeof setInitialTestState;
|
|
1702
1704
|
};
|
|
1703
1705
|
|
|
1704
1706
|
declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
|
|
@@ -2109,7 +2111,7 @@ declare global {
|
|
|
2109
2111
|
* not configured, nothing will be traced. If this is not an `OpenAI` object, this function is
|
|
2110
2112
|
* a no-op.
|
|
2111
2113
|
*
|
|
2112
|
-
* Currently, this
|
|
2114
|
+
* Currently, this supports both the `v4` and `v5` API.
|
|
2113
2115
|
*
|
|
2114
2116
|
* @param openai
|
|
2115
2117
|
* @returns The wrapped `OpenAI` object.
|
package/dist/browser.d.ts
CHANGED
|
@@ -1690,6 +1690,7 @@ interface DatasetSummary {
|
|
|
1690
1690
|
datasetUrl: string;
|
|
1691
1691
|
dataSummary: DataSummary | undefined;
|
|
1692
1692
|
}
|
|
1693
|
+
declare function setInitialTestState(): void;
|
|
1693
1694
|
declare function simulateLoginForTests(): Promise<BraintrustState>;
|
|
1694
1695
|
declare function simulateLogoutForTests(): BraintrustState;
|
|
1695
1696
|
declare const _exportsForTestingOnly: {
|
|
@@ -1699,6 +1700,7 @@ declare const _exportsForTestingOnly: {
|
|
|
1699
1700
|
clearTestBackgroundLogger: typeof clearTestBackgroundLogger;
|
|
1700
1701
|
simulateLoginForTests: typeof simulateLoginForTests;
|
|
1701
1702
|
simulateLogoutForTests: typeof simulateLogoutForTests;
|
|
1703
|
+
setInitialTestState: typeof setInitialTestState;
|
|
1702
1704
|
};
|
|
1703
1705
|
|
|
1704
1706
|
declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
|
|
@@ -2109,7 +2111,7 @@ declare global {
|
|
|
2109
2111
|
* not configured, nothing will be traced. If this is not an `OpenAI` object, this function is
|
|
2110
2112
|
* a no-op.
|
|
2111
2113
|
*
|
|
2112
|
-
* Currently, this
|
|
2114
|
+
* Currently, this supports both the `v4` and `v5` API.
|
|
2113
2115
|
*
|
|
2114
2116
|
* @param openai
|
|
2115
2117
|
* @returns The wrapped `OpenAI` object.
|
package/dist/browser.js
CHANGED
|
@@ -4563,6 +4563,11 @@ var Prompt = (_class13 = class _Prompt {
|
|
|
4563
4563
|
}
|
|
4564
4564
|
}, _class13);
|
|
4565
4565
|
var TEST_API_KEY = "___TEST_API_KEY__THIS_IS_NOT_REAL___";
|
|
4566
|
+
function setInitialTestState() {
|
|
4567
|
+
if (!_internalGetGlobalState()) {
|
|
4568
|
+
_internalSetInitialState();
|
|
4569
|
+
}
|
|
4570
|
+
}
|
|
4566
4571
|
async function simulateLoginForTests() {
|
|
4567
4572
|
return await login({
|
|
4568
4573
|
apiKey: TEST_API_KEY,
|
|
@@ -4580,7 +4585,8 @@ var _exportsForTestingOnly = {
|
|
|
4580
4585
|
useTestBackgroundLogger,
|
|
4581
4586
|
clearTestBackgroundLogger,
|
|
4582
4587
|
simulateLoginForTests,
|
|
4583
|
-
simulateLogoutForTests
|
|
4588
|
+
simulateLogoutForTests,
|
|
4589
|
+
setInitialTestState
|
|
4584
4590
|
};
|
|
4585
4591
|
|
|
4586
4592
|
// src/browser-config.ts
|
|
@@ -5030,10 +5036,17 @@ function wrapOpenAI(openai) {
|
|
|
5030
5036
|
}
|
|
5031
5037
|
globalThis.__inherited_braintrust_wrap_openai = wrapOpenAI;
|
|
5032
5038
|
function wrapOpenAIv4(openai) {
|
|
5033
|
-
const completionProxy =
|
|
5034
|
-
|
|
5035
|
-
|
|
5036
|
-
|
|
5039
|
+
const completionProxy = new Proxy(openai.chat.completions, {
|
|
5040
|
+
get(target, name, receiver) {
|
|
5041
|
+
const baseVal = Reflect.get(target, name, receiver);
|
|
5042
|
+
if (name === "create") {
|
|
5043
|
+
return wrapChatCompletion(baseVal.bind(target));
|
|
5044
|
+
} else if (name === "parse") {
|
|
5045
|
+
return wrapBetaChatCompletionParse(baseVal.bind(target));
|
|
5046
|
+
}
|
|
5047
|
+
return baseVal;
|
|
5048
|
+
}
|
|
5049
|
+
});
|
|
5037
5050
|
const chatProxy = new Proxy(openai.chat, {
|
|
5038
5051
|
get(target, name, receiver) {
|
|
5039
5052
|
if (name === "completions") {
|
|
@@ -5427,6 +5440,8 @@ var evalBodySchema = _zod.z.object({
|
|
|
5427
5440
|
name: _zod.z.string()
|
|
5428
5441
|
})
|
|
5429
5442
|
).nullish(),
|
|
5443
|
+
experiment_name: _zod.z.string().nullish(),
|
|
5444
|
+
project_id: _zod.z.string().nullish(),
|
|
5430
5445
|
parent: _typespecs.invokeParent.optional(),
|
|
5431
5446
|
stream: _zod.z.boolean().optional()
|
|
5432
5447
|
});
|
package/dist/browser.mjs
CHANGED
|
@@ -4563,6 +4563,11 @@ var Prompt = class _Prompt {
|
|
|
4563
4563
|
}
|
|
4564
4564
|
};
|
|
4565
4565
|
var TEST_API_KEY = "___TEST_API_KEY__THIS_IS_NOT_REAL___";
|
|
4566
|
+
function setInitialTestState() {
|
|
4567
|
+
if (!_internalGetGlobalState()) {
|
|
4568
|
+
_internalSetInitialState();
|
|
4569
|
+
}
|
|
4570
|
+
}
|
|
4566
4571
|
async function simulateLoginForTests() {
|
|
4567
4572
|
return await login({
|
|
4568
4573
|
apiKey: TEST_API_KEY,
|
|
@@ -4580,7 +4585,8 @@ var _exportsForTestingOnly = {
|
|
|
4580
4585
|
useTestBackgroundLogger,
|
|
4581
4586
|
clearTestBackgroundLogger,
|
|
4582
4587
|
simulateLoginForTests,
|
|
4583
|
-
simulateLogoutForTests
|
|
4588
|
+
simulateLogoutForTests,
|
|
4589
|
+
setInitialTestState
|
|
4584
4590
|
};
|
|
4585
4591
|
|
|
4586
4592
|
// src/browser-config.ts
|
|
@@ -5030,10 +5036,17 @@ function wrapOpenAI(openai) {
|
|
|
5030
5036
|
}
|
|
5031
5037
|
globalThis.__inherited_braintrust_wrap_openai = wrapOpenAI;
|
|
5032
5038
|
function wrapOpenAIv4(openai) {
|
|
5033
|
-
const completionProxy =
|
|
5034
|
-
|
|
5035
|
-
|
|
5036
|
-
|
|
5039
|
+
const completionProxy = new Proxy(openai.chat.completions, {
|
|
5040
|
+
get(target, name, receiver) {
|
|
5041
|
+
const baseVal = Reflect.get(target, name, receiver);
|
|
5042
|
+
if (name === "create") {
|
|
5043
|
+
return wrapChatCompletion(baseVal.bind(target));
|
|
5044
|
+
} else if (name === "parse") {
|
|
5045
|
+
return wrapBetaChatCompletionParse(baseVal.bind(target));
|
|
5046
|
+
}
|
|
5047
|
+
return baseVal;
|
|
5048
|
+
}
|
|
5049
|
+
});
|
|
5037
5050
|
const chatProxy = new Proxy(openai.chat, {
|
|
5038
5051
|
get(target, name, receiver) {
|
|
5039
5052
|
if (name === "completions") {
|
|
@@ -5427,6 +5440,8 @@ var evalBodySchema = z3.object({
|
|
|
5427
5440
|
name: z3.string()
|
|
5428
5441
|
})
|
|
5429
5442
|
).nullish(),
|
|
5443
|
+
experiment_name: z3.string().nullish(),
|
|
5444
|
+
project_id: z3.string().nullish(),
|
|
5430
5445
|
parent: invokeParent.optional(),
|
|
5431
5446
|
stream: z3.boolean().optional()
|
|
5432
5447
|
});
|
package/dist/cli.js
CHANGED
|
@@ -1232,7 +1232,7 @@ var require_package = __commonJS({
|
|
|
1232
1232
|
"package.json"(exports2, module2) {
|
|
1233
1233
|
module2.exports = {
|
|
1234
1234
|
name: "braintrust",
|
|
1235
|
-
version: "0.2.
|
|
1235
|
+
version: "0.2.1",
|
|
1236
1236
|
description: "SDK for integrating Braintrust",
|
|
1237
1237
|
repository: {
|
|
1238
1238
|
type: "git",
|
|
@@ -1283,11 +1283,13 @@ var require_package = __commonJS({
|
|
|
1283
1283
|
docs: "npx typedoc --options typedoc.json src/index.ts",
|
|
1284
1284
|
prepublishOnly: "../../scripts/node_prepublish_sdk.py",
|
|
1285
1285
|
postpublish: "../../scripts/node_postpublish_sdk.py",
|
|
1286
|
-
test: "vitest run --exclude src/wrappers/anthropic.test.ts --exclude src/wrappers/oai.test.ts --exclude src/otel.test.ts --exclude src/otel-no-deps.test.ts",
|
|
1286
|
+
test: "vitest run --exclude src/wrappers/anthropic.test.ts --exclude src/wrappers/oai.test.ts --exclude src/otel.test.ts --exclude src/otel-no-deps.test.ts --exclude src/wrappers/ai-sdk-v1.test.ts --exclude src/wrappers/ai-sdk-v2.test.ts",
|
|
1287
1287
|
"test:anthropic": "vitest run src/wrappers/anthropic.test.ts",
|
|
1288
1288
|
"test:openai": "vitest run src/wrappers/oai.test.ts",
|
|
1289
1289
|
"test:otel": "vitest run src/otel.test.ts",
|
|
1290
|
-
"test:otel-no-deps": "vitest run src/otel-no-deps.test.ts --reporter=verbose"
|
|
1290
|
+
"test:otel-no-deps": "vitest run src/otel-no-deps.test.ts --reporter=verbose",
|
|
1291
|
+
"test:ai-sdk-v1": "vitest run src/wrappers/ai-sdk-v1.test.ts",
|
|
1292
|
+
"test:ai-sdk-v2": "vitest run src/wrappers/ai-sdk-v2.test.ts src/wrappers/ai-sdk-v1.test.ts"
|
|
1291
1293
|
},
|
|
1292
1294
|
author: "",
|
|
1293
1295
|
license: "MIT",
|
|
@@ -8875,6 +8877,8 @@ var evalBodySchema = import_zod8.z.object({
|
|
|
8875
8877
|
name: import_zod8.z.string()
|
|
8876
8878
|
})
|
|
8877
8879
|
).nullish(),
|
|
8880
|
+
experiment_name: import_zod8.z.string().nullish(),
|
|
8881
|
+
project_id: import_zod8.z.string().nullish(),
|
|
8878
8882
|
parent: import_typespecs6.invokeParent.optional(),
|
|
8879
8883
|
stream: import_zod8.z.boolean().optional()
|
|
8880
8884
|
});
|
|
@@ -8957,7 +8961,16 @@ function runDevServer(evaluators, opts) {
|
|
|
8957
8961
|
"/eval",
|
|
8958
8962
|
checkAuthorized,
|
|
8959
8963
|
asyncHandler(async (req, res) => {
|
|
8960
|
-
const {
|
|
8964
|
+
const {
|
|
8965
|
+
name,
|
|
8966
|
+
parameters,
|
|
8967
|
+
parent,
|
|
8968
|
+
experiment_name,
|
|
8969
|
+
project_id,
|
|
8970
|
+
data,
|
|
8971
|
+
scores,
|
|
8972
|
+
stream
|
|
8973
|
+
} = evalBodySchema.parse(req.body);
|
|
8961
8974
|
const state = await cachedLogin({ apiKey: req.ctx?.token });
|
|
8962
8975
|
const evaluator = allEvaluators[name];
|
|
8963
8976
|
if (!evaluator) {
|
|
@@ -9016,17 +9029,19 @@ function runDevServer(evaluators, opts) {
|
|
|
9016
9029
|
) ?? []
|
|
9017
9030
|
),
|
|
9018
9031
|
task,
|
|
9019
|
-
state
|
|
9032
|
+
state,
|
|
9033
|
+
experimentName: experiment_name ?? void 0,
|
|
9034
|
+
projectId: project_id ?? void 0
|
|
9020
9035
|
},
|
|
9021
9036
|
{
|
|
9022
9037
|
// Avoid printing the bar to the console.
|
|
9023
9038
|
progress: {
|
|
9024
|
-
start: (
|
|
9039
|
+
start: () => {
|
|
9025
9040
|
},
|
|
9026
9041
|
stop: () => {
|
|
9027
9042
|
console.log("Finished running experiment");
|
|
9028
9043
|
},
|
|
9029
|
-
increment: (
|
|
9044
|
+
increment: () => {
|
|
9030
9045
|
}
|
|
9031
9046
|
},
|
|
9032
9047
|
stream: (data2) => {
|
package/dist/index.d.mts
CHANGED
|
@@ -1691,6 +1691,7 @@ interface DatasetSummary {
|
|
|
1691
1691
|
datasetUrl: string;
|
|
1692
1692
|
dataSummary: DataSummary | undefined;
|
|
1693
1693
|
}
|
|
1694
|
+
declare function setInitialTestState(): void;
|
|
1694
1695
|
declare function simulateLoginForTests(): Promise<BraintrustState>;
|
|
1695
1696
|
declare function simulateLogoutForTests(): BraintrustState;
|
|
1696
1697
|
declare const _exportsForTestingOnly: {
|
|
@@ -1700,6 +1701,7 @@ declare const _exportsForTestingOnly: {
|
|
|
1700
1701
|
clearTestBackgroundLogger: typeof clearTestBackgroundLogger;
|
|
1701
1702
|
simulateLoginForTests: typeof simulateLoginForTests;
|
|
1702
1703
|
simulateLogoutForTests: typeof simulateLogoutForTests;
|
|
1704
|
+
setInitialTestState: typeof setInitialTestState;
|
|
1703
1705
|
};
|
|
1704
1706
|
|
|
1705
1707
|
declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
|
|
@@ -7134,7 +7136,7 @@ declare global {
|
|
|
7134
7136
|
* not configured, nothing will be traced. If this is not an `OpenAI` object, this function is
|
|
7135
7137
|
* a no-op.
|
|
7136
7138
|
*
|
|
7137
|
-
* Currently, this
|
|
7139
|
+
* Currently, this supports both the `v4` and `v5` API.
|
|
7138
7140
|
*
|
|
7139
7141
|
* @param openai
|
|
7140
7142
|
* @returns The wrapped `OpenAI` object.
|
|
@@ -7145,6 +7147,50 @@ declare const LEGACY_CACHED_HEADER = "x-cached";
|
|
|
7145
7147
|
declare const X_CACHED_HEADER = "x-bt-cached";
|
|
7146
7148
|
declare function parseCachedHeader(value: string | null | undefined): number | undefined;
|
|
7147
7149
|
|
|
7150
|
+
interface LanguageModelV2Middleware<TModel = any, TCallOptions = any> {
|
|
7151
|
+
wrapGenerate?: (options: {
|
|
7152
|
+
doGenerate: () => any;
|
|
7153
|
+
doStream: () => any;
|
|
7154
|
+
params: TCallOptions;
|
|
7155
|
+
model: TModel;
|
|
7156
|
+
}) => Promise<any>;
|
|
7157
|
+
wrapStream?: (options: {
|
|
7158
|
+
doGenerate: () => any;
|
|
7159
|
+
doStream: () => any;
|
|
7160
|
+
params: TCallOptions;
|
|
7161
|
+
model: TModel;
|
|
7162
|
+
}) => Promise<any>;
|
|
7163
|
+
}
|
|
7164
|
+
/**
|
|
7165
|
+
* Configuration options for the AI SDK middleware
|
|
7166
|
+
*/
|
|
7167
|
+
interface MiddlewareConfig {
|
|
7168
|
+
/** Enable debug logging */
|
|
7169
|
+
debug?: boolean;
|
|
7170
|
+
/** Name identifier for the middleware instance */
|
|
7171
|
+
name?: string;
|
|
7172
|
+
}
|
|
7173
|
+
/**
|
|
7174
|
+
* Creates a Braintrust middleware for AI SDK v2 that automatically traces
|
|
7175
|
+
* generateText and streamText calls with comprehensive metadata and metrics.
|
|
7176
|
+
*
|
|
7177
|
+
* @param config - Configuration options for the middleware
|
|
7178
|
+
* @returns A middleware object compatible with AI SDK v2's wrapLanguageModel
|
|
7179
|
+
*
|
|
7180
|
+
* @example
|
|
7181
|
+
* ```typescript
|
|
7182
|
+
* import { wrapLanguageModel } from "ai";
|
|
7183
|
+
* import { openai } from "@ai-sdk/openai";
|
|
7184
|
+
* import { BraintrustMiddleware } from "braintrust";
|
|
7185
|
+
*
|
|
7186
|
+
* const model = wrapLanguageModel({
|
|
7187
|
+
* model: openai("gpt-4"),
|
|
7188
|
+
* middleware: BraintrustMiddleware({ debug: true, name: "MyMiddleware" })
|
|
7189
|
+
* });
|
|
7190
|
+
* ```
|
|
7191
|
+
*/
|
|
7192
|
+
declare function BraintrustMiddleware(config?: MiddlewareConfig): LanguageModelV2Middleware<any, any>;
|
|
7193
|
+
|
|
7148
7194
|
/**
|
|
7149
7195
|
* Wrap an ai-sdk model (created with `.chat()`, `.completion()`, etc.) to add tracing. If Braintrust is
|
|
7150
7196
|
* not configured, this is a no-op
|
|
@@ -7406,6 +7452,7 @@ declare const braintrust_BaseExperiment: typeof BaseExperiment;
|
|
|
7406
7452
|
type braintrust_BaseMetadata = BaseMetadata;
|
|
7407
7453
|
type braintrust_BraintrustExporter = BraintrustExporter;
|
|
7408
7454
|
declare const braintrust_BraintrustExporter: typeof BraintrustExporter;
|
|
7455
|
+
declare const braintrust_BraintrustMiddleware: typeof BraintrustMiddleware;
|
|
7409
7456
|
type braintrust_BraintrustSpanProcessor = BraintrustSpanProcessor;
|
|
7410
7457
|
declare const braintrust_BraintrustSpanProcessor: typeof BraintrustSpanProcessor;
|
|
7411
7458
|
type braintrust_BraintrustState = BraintrustState;
|
|
@@ -7564,7 +7611,7 @@ declare const braintrust_wrapOpenAI: typeof wrapOpenAI;
|
|
|
7564
7611
|
declare const braintrust_wrapOpenAIv4: typeof wrapOpenAIv4;
|
|
7565
7612
|
declare const braintrust_wrapTraced: typeof wrapTraced;
|
|
7566
7613
|
declare namespace braintrust {
|
|
7567
|
-
export { braintrust_AISpanProcessor as AISpanProcessor, type braintrust_AnyDataset as AnyDataset, braintrust_Attachment as Attachment, type braintrust_AttachmentParams as AttachmentParams, type braintrust_BackgroundLoggerOpts as BackgroundLoggerOpts, braintrust_BaseAttachment as BaseAttachment, braintrust_BaseExperiment as BaseExperiment, type braintrust_BaseMetadata as BaseMetadata, braintrust_BraintrustExporter as BraintrustExporter, braintrust_BraintrustSpanProcessor as BraintrustSpanProcessor, braintrust_BraintrustState as BraintrustState, braintrust_BraintrustStream as BraintrustStream, type braintrust_BraintrustStreamChunk as BraintrustStreamChunk, type braintrust_ChatPrompt as ChatPrompt, braintrust_CodeFunction as CodeFunction, type braintrust_CodeOpts as CodeOpts, braintrust_CodePrompt as CodePrompt, type braintrust_CompiledPrompt as CompiledPrompt, type braintrust_CompiledPromptParams as CompiledPromptParams, type braintrust_CompletionPrompt as CompletionPrompt, type braintrust_CreateProjectOpts as CreateProjectOpts, type braintrust_DataSummary as DataSummary, braintrust_Dataset as Dataset, type braintrust_DatasetSummary as DatasetSummary, type braintrust_DefaultMetadataType as DefaultMetadataType, type braintrust_DefaultPromptArgs as DefaultPromptArgs, braintrust_ERR_PERMALINK as ERR_PERMALINK, type braintrust_EndSpanArgs as EndSpanArgs, braintrust_Eval as Eval, type braintrust_EvalCase as EvalCase, type braintrust_EvalHooks as EvalHooks, type braintrust_EvalResult as EvalResult, braintrust_EvalResultWithSummary as EvalResultWithSummary, type braintrust_EvalScorer as EvalScorer, type braintrust_EvalScorerArgs as EvalScorerArgs, type braintrust_EvalTask as EvalTask, type braintrust_Evaluator as Evaluator, type braintrust_EvaluatorDef as EvaluatorDef, type braintrust_EvaluatorFile as EvaluatorFile, braintrust_Experiment as Experiment, type braintrust_ExperimentSummary as ExperimentSummary, type braintrust_Exportable as Exportable, braintrust_ExternalAttachment as ExternalAttachment, type braintrust_ExternalAttachmentParams as ExternalAttachmentParams, braintrust_FailedHTTPResponse as FailedHTTPResponse, type braintrust_FullInitOptions as FullInitOptions, type braintrust_FullLoginOptions as FullLoginOptions, type braintrust_FunctionEvent as FunctionEvent, braintrust_INTERNAL_BTQL_LIMIT as INTERNAL_BTQL_LIMIT, type braintrust_InitOptions as InitOptions, type braintrust_InvokeFunctionArgs as InvokeFunctionArgs, type braintrust_InvokeReturn as InvokeReturn, braintrust_LEGACY_CACHED_HEADER as LEGACY_CACHED_HEADER, braintrust_LazyValue as LazyValue, type braintrust_LogOptions as LogOptions, braintrust_Logger as Logger, type braintrust_LoginOptions as LoginOptions, type braintrust_MetricSummary as MetricSummary, braintrust_NOOP_SPAN as NOOP_SPAN, braintrust_NOOP_SPAN_PERMALINK as NOOP_SPAN_PERMALINK, braintrust_NoopSpan as NoopSpan, type braintrust_ObjectMetadata as ObjectMetadata, braintrust_Project as Project, braintrust_ProjectNameIdMap as ProjectNameIdMap, type braintrust_PromiseUnless as PromiseUnless, braintrust_Prompt as Prompt, braintrust_PromptBuilder as PromptBuilder, type braintrust_PromptContents as PromptContents, type braintrust_PromptDefinition as PromptDefinition, type braintrust_PromptDefinitionWithTools as PromptDefinitionWithTools, type braintrust_PromptOpts as PromptOpts, type braintrust_PromptRowWithId as PromptRowWithId, braintrust_ReadonlyAttachment as ReadonlyAttachment, braintrust_ReadonlyExperiment as ReadonlyExperiment, braintrust_Reporter as Reporter, type braintrust_ReporterBody as ReporterBody, type braintrust_ScoreSummary as ScoreSummary, braintrust_ScorerBuilder as ScorerBuilder, type braintrust_ScorerOpts as ScorerOpts, type braintrust_SerializedBraintrustState as SerializedBraintrustState, type braintrust_SetCurrentArg as SetCurrentArg, type Span$1 as Span, type braintrust_SpanContext as SpanContext, braintrust_SpanImpl as SpanImpl, type braintrust_StartSpanArgs as StartSpanArgs, braintrust_TestBackgroundLogger as TestBackgroundLogger, braintrust_ToolBuilder as ToolBuilder, type braintrust_WithTransactionId as WithTransactionId, braintrust_X_CACHED_HEADER as X_CACHED_HEADER, braintrust__exportsForTestingOnly as _exportsForTestingOnly, braintrust__internalGetGlobalState as _internalGetGlobalState, braintrust__internalSetInitialState as _internalSetInitialState, braintrust_braintrustStreamChunkSchema as braintrustStreamChunkSchema, braintrust_buildLocalSummary as buildLocalSummary, braintrust_createFinalValuePassThroughStream as createFinalValuePassThroughStream, braintrust_currentExperiment as currentExperiment, braintrust_currentLogger as currentLogger, braintrust_currentSpan as currentSpan, braintrust_defaultErrorScoreHandler as defaultErrorScoreHandler, braintrust_deserializePlainStringAsJSON as deserializePlainStringAsJSON, braintrust_devNullWritableStream as devNullWritableStream, braintrust_flush as flush, braintrust_getSpanParentObject as getSpanParentObject, graphFramework as graph, braintrust_init as init, braintrust_initDataset as initDataset, braintrust_initExperiment as initExperiment, braintrust_initFunction as initFunction, braintrust_initLogger as initLogger, braintrust_invoke as invoke, braintrust_loadPrompt as loadPrompt, braintrust_log as log, braintrust_logError as logError, braintrust_login as login, braintrust_loginToState as loginToState, braintrust_newId as newId, braintrust_parseCachedHeader as parseCachedHeader, braintrust_permalink as permalink, braintrust_projects as projects, braintrust_promptContentsSchema as promptContentsSchema, braintrust_promptDefinitionSchema as promptDefinitionSchema, braintrust_promptDefinitionToPromptData as promptDefinitionToPromptData, braintrust_promptDefinitionWithToolsSchema as promptDefinitionWithToolsSchema, braintrust_renderMessage as renderMessage, braintrust_renderPromptParams as renderPromptParams, braintrust_reportFailures as reportFailures, braintrust_runEvaluator as runEvaluator, braintrust_setFetch as setFetch, braintrust_spanComponentsToObjectId as spanComponentsToObjectId, braintrust_startSpan as startSpan, braintrust_summarize as summarize, braintrust_toolFunctionDefinitionSchema as toolFunctionDefinitionSchema, braintrust_traceable as traceable, braintrust_traced as traced, braintrust_updateSpan as updateSpan, braintrust_withCurrent as withCurrent, braintrust_withDataset as withDataset, braintrust_withExperiment as withExperiment, braintrust_withLogger as withLogger, braintrust_withParent as withParent, braintrust_wrapAISDKModel as wrapAISDKModel, braintrust_wrapAnthropic as wrapAnthropic, braintrust_wrapOpenAI as wrapOpenAI, braintrust_wrapOpenAIv4 as wrapOpenAIv4, braintrust_wrapTraced as wrapTraced };
|
|
7614
|
+
export { braintrust_AISpanProcessor as AISpanProcessor, type braintrust_AnyDataset as AnyDataset, braintrust_Attachment as Attachment, type braintrust_AttachmentParams as AttachmentParams, type braintrust_BackgroundLoggerOpts as BackgroundLoggerOpts, braintrust_BaseAttachment as BaseAttachment, braintrust_BaseExperiment as BaseExperiment, type braintrust_BaseMetadata as BaseMetadata, braintrust_BraintrustExporter as BraintrustExporter, braintrust_BraintrustMiddleware as BraintrustMiddleware, braintrust_BraintrustSpanProcessor as BraintrustSpanProcessor, braintrust_BraintrustState as BraintrustState, braintrust_BraintrustStream as BraintrustStream, type braintrust_BraintrustStreamChunk as BraintrustStreamChunk, type braintrust_ChatPrompt as ChatPrompt, braintrust_CodeFunction as CodeFunction, type braintrust_CodeOpts as CodeOpts, braintrust_CodePrompt as CodePrompt, type braintrust_CompiledPrompt as CompiledPrompt, type braintrust_CompiledPromptParams as CompiledPromptParams, type braintrust_CompletionPrompt as CompletionPrompt, type braintrust_CreateProjectOpts as CreateProjectOpts, type braintrust_DataSummary as DataSummary, braintrust_Dataset as Dataset, type braintrust_DatasetSummary as DatasetSummary, type braintrust_DefaultMetadataType as DefaultMetadataType, type braintrust_DefaultPromptArgs as DefaultPromptArgs, braintrust_ERR_PERMALINK as ERR_PERMALINK, type braintrust_EndSpanArgs as EndSpanArgs, braintrust_Eval as Eval, type braintrust_EvalCase as EvalCase, type braintrust_EvalHooks as EvalHooks, type braintrust_EvalResult as EvalResult, braintrust_EvalResultWithSummary as EvalResultWithSummary, type braintrust_EvalScorer as EvalScorer, type braintrust_EvalScorerArgs as EvalScorerArgs, type braintrust_EvalTask as EvalTask, type braintrust_Evaluator as Evaluator, type braintrust_EvaluatorDef as EvaluatorDef, type braintrust_EvaluatorFile as EvaluatorFile, braintrust_Experiment as Experiment, type braintrust_ExperimentSummary as ExperimentSummary, type braintrust_Exportable as Exportable, braintrust_ExternalAttachment as ExternalAttachment, type braintrust_ExternalAttachmentParams as ExternalAttachmentParams, braintrust_FailedHTTPResponse as FailedHTTPResponse, type braintrust_FullInitOptions as FullInitOptions, type braintrust_FullLoginOptions as FullLoginOptions, type braintrust_FunctionEvent as FunctionEvent, braintrust_INTERNAL_BTQL_LIMIT as INTERNAL_BTQL_LIMIT, type braintrust_InitOptions as InitOptions, type braintrust_InvokeFunctionArgs as InvokeFunctionArgs, type braintrust_InvokeReturn as InvokeReturn, braintrust_LEGACY_CACHED_HEADER as LEGACY_CACHED_HEADER, braintrust_LazyValue as LazyValue, type braintrust_LogOptions as LogOptions, braintrust_Logger as Logger, type braintrust_LoginOptions as LoginOptions, type braintrust_MetricSummary as MetricSummary, braintrust_NOOP_SPAN as NOOP_SPAN, braintrust_NOOP_SPAN_PERMALINK as NOOP_SPAN_PERMALINK, braintrust_NoopSpan as NoopSpan, type braintrust_ObjectMetadata as ObjectMetadata, braintrust_Project as Project, braintrust_ProjectNameIdMap as ProjectNameIdMap, type braintrust_PromiseUnless as PromiseUnless, braintrust_Prompt as Prompt, braintrust_PromptBuilder as PromptBuilder, type braintrust_PromptContents as PromptContents, type braintrust_PromptDefinition as PromptDefinition, type braintrust_PromptDefinitionWithTools as PromptDefinitionWithTools, type braintrust_PromptOpts as PromptOpts, type braintrust_PromptRowWithId as PromptRowWithId, braintrust_ReadonlyAttachment as ReadonlyAttachment, braintrust_ReadonlyExperiment as ReadonlyExperiment, braintrust_Reporter as Reporter, type braintrust_ReporterBody as ReporterBody, type braintrust_ScoreSummary as ScoreSummary, braintrust_ScorerBuilder as ScorerBuilder, type braintrust_ScorerOpts as ScorerOpts, type braintrust_SerializedBraintrustState as SerializedBraintrustState, type braintrust_SetCurrentArg as SetCurrentArg, type Span$1 as Span, type braintrust_SpanContext as SpanContext, braintrust_SpanImpl as SpanImpl, type braintrust_StartSpanArgs as StartSpanArgs, braintrust_TestBackgroundLogger as TestBackgroundLogger, braintrust_ToolBuilder as ToolBuilder, type braintrust_WithTransactionId as WithTransactionId, braintrust_X_CACHED_HEADER as X_CACHED_HEADER, braintrust__exportsForTestingOnly as _exportsForTestingOnly, braintrust__internalGetGlobalState as _internalGetGlobalState, braintrust__internalSetInitialState as _internalSetInitialState, braintrust_braintrustStreamChunkSchema as braintrustStreamChunkSchema, braintrust_buildLocalSummary as buildLocalSummary, braintrust_createFinalValuePassThroughStream as createFinalValuePassThroughStream, braintrust_currentExperiment as currentExperiment, braintrust_currentLogger as currentLogger, braintrust_currentSpan as currentSpan, braintrust_defaultErrorScoreHandler as defaultErrorScoreHandler, braintrust_deserializePlainStringAsJSON as deserializePlainStringAsJSON, braintrust_devNullWritableStream as devNullWritableStream, braintrust_flush as flush, braintrust_getSpanParentObject as getSpanParentObject, graphFramework as graph, braintrust_init as init, braintrust_initDataset as initDataset, braintrust_initExperiment as initExperiment, braintrust_initFunction as initFunction, braintrust_initLogger as initLogger, braintrust_invoke as invoke, braintrust_loadPrompt as loadPrompt, braintrust_log as log, braintrust_logError as logError, braintrust_login as login, braintrust_loginToState as loginToState, braintrust_newId as newId, braintrust_parseCachedHeader as parseCachedHeader, braintrust_permalink as permalink, braintrust_projects as projects, braintrust_promptContentsSchema as promptContentsSchema, braintrust_promptDefinitionSchema as promptDefinitionSchema, braintrust_promptDefinitionToPromptData as promptDefinitionToPromptData, braintrust_promptDefinitionWithToolsSchema as promptDefinitionWithToolsSchema, braintrust_renderMessage as renderMessage, braintrust_renderPromptParams as renderPromptParams, braintrust_reportFailures as reportFailures, braintrust_runEvaluator as runEvaluator, braintrust_setFetch as setFetch, braintrust_spanComponentsToObjectId as spanComponentsToObjectId, braintrust_startSpan as startSpan, braintrust_summarize as summarize, braintrust_toolFunctionDefinitionSchema as toolFunctionDefinitionSchema, braintrust_traceable as traceable, braintrust_traced as traced, braintrust_updateSpan as updateSpan, braintrust_withCurrent as withCurrent, braintrust_withDataset as withDataset, braintrust_withExperiment as withExperiment, braintrust_withLogger as withLogger, braintrust_withParent as withParent, braintrust_wrapAISDKModel as wrapAISDKModel, braintrust_wrapAnthropic as wrapAnthropic, braintrust_wrapOpenAI as wrapOpenAI, braintrust_wrapOpenAIv4 as wrapOpenAIv4, braintrust_wrapTraced as wrapTraced };
|
|
7568
7615
|
}
|
|
7569
7616
|
|
|
7570
7617
|
type EvaluatorManifest = Record<string, EvaluatorDef<unknown, unknown, unknown, BaseMetadata>>;
|
|
@@ -14792,4 +14839,4 @@ type EvaluatorDefinitions = z.infer<typeof evaluatorDefinitionsSchema>;
|
|
|
14792
14839
|
* @module braintrust
|
|
14793
14840
|
*/
|
|
14794
14841
|
|
|
14795
|
-
export { AISpanProcessor, type AnyDataset, Attachment, type AttachmentParams, type BackgroundLoggerOpts, BaseAttachment, BaseExperiment, type BaseMetadata, BraintrustExporter, BraintrustSpanProcessor, BraintrustState, BraintrustStream, type BraintrustStreamChunk, type ChatPrompt, CodeFunction, type CodeOpts, CodePrompt, type CompiledPrompt, type CompiledPromptParams, type CompletionPrompt, type CreateProjectOpts, type DataSummary, Dataset, type DatasetSummary, type DefaultMetadataType, type DefaultPromptArgs, ERR_PERMALINK, type EndSpanArgs, Eval, type EvalCase, type EvalHooks, type EvalParameterSerializedSchema, type EvalParameters, type EvalResult, EvalResultWithSummary, type EvalScorer, type EvalScorerArgs, type EvalTask, type Evaluator, type EvaluatorDef, type EvaluatorDefinition, type EvaluatorDefinitions, type EvaluatorFile, type EvaluatorManifest, Experiment, type ExperimentSummary, type Exportable, ExternalAttachment, type ExternalAttachmentParams, FailedHTTPResponse, type FullInitOptions, type FullLoginOptions, type FunctionEvent, INTERNAL_BTQL_LIMIT, type InitOptions, type InvokeFunctionArgs, type InvokeReturn, LEGACY_CACHED_HEADER, LazyValue, type LogOptions, Logger, type LoginOptions, type MetricSummary, NOOP_SPAN, NOOP_SPAN_PERMALINK, NoopSpan, type ObjectMetadata, Project, ProjectNameIdMap, type PromiseUnless, Prompt, PromptBuilder, type PromptContents, type PromptDefinition, type PromptDefinitionWithTools, type PromptOpts, type PromptRowWithId, ReadonlyAttachment, ReadonlyExperiment, Reporter, type ReporterBody, type ScoreSummary, ScorerBuilder, type ScorerOpts, type SerializedBraintrustState, type SetCurrentArg, type Span$1 as Span, type SpanContext, SpanImpl, type StartSpanArgs, TestBackgroundLogger, ToolBuilder, type WithTransactionId, X_CACHED_HEADER, _exportsForTestingOnly, _internalGetGlobalState, _internalSetInitialState, braintrustStreamChunkSchema, buildLocalSummary, createFinalValuePassThroughStream, currentExperiment, currentLogger, currentSpan, braintrust as default, defaultErrorScoreHandler, deserializePlainStringAsJSON, devNullWritableStream, evaluatorDefinitionSchema, evaluatorDefinitionsSchema, flush, getSpanParentObject, graphFramework as graph, init, initDataset, initExperiment, initFunction, initLogger, invoke, loadPrompt, log, logError, login, loginToState, newId, parseCachedHeader, permalink, projects, promptContentsSchema, promptDefinitionSchema, promptDefinitionToPromptData, promptDefinitionWithToolsSchema, renderMessage, renderPromptParams, reportFailures, runEvaluator, setFetch, spanComponentsToObjectId, startSpan, summarize, traceable, traced, updateSpan, withCurrent, withDataset, withExperiment, withLogger, withParent, wrapAISDKModel, wrapAnthropic, wrapOpenAI, wrapOpenAIv4, wrapTraced };
|
|
14842
|
+
export { AISpanProcessor, type AnyDataset, Attachment, type AttachmentParams, type BackgroundLoggerOpts, BaseAttachment, BaseExperiment, type BaseMetadata, BraintrustExporter, BraintrustMiddleware, BraintrustSpanProcessor, BraintrustState, BraintrustStream, type BraintrustStreamChunk, type ChatPrompt, CodeFunction, type CodeOpts, CodePrompt, type CompiledPrompt, type CompiledPromptParams, type CompletionPrompt, type CreateProjectOpts, type DataSummary, Dataset, type DatasetSummary, type DefaultMetadataType, type DefaultPromptArgs, ERR_PERMALINK, type EndSpanArgs, Eval, type EvalCase, type EvalHooks, type EvalParameterSerializedSchema, type EvalParameters, type EvalResult, EvalResultWithSummary, type EvalScorer, type EvalScorerArgs, type EvalTask, type Evaluator, type EvaluatorDef, type EvaluatorDefinition, type EvaluatorDefinitions, type EvaluatorFile, type EvaluatorManifest, Experiment, type ExperimentSummary, type Exportable, ExternalAttachment, type ExternalAttachmentParams, FailedHTTPResponse, type FullInitOptions, type FullLoginOptions, type FunctionEvent, INTERNAL_BTQL_LIMIT, type InitOptions, type InvokeFunctionArgs, type InvokeReturn, LEGACY_CACHED_HEADER, LazyValue, type LogOptions, Logger, type LoginOptions, type MetricSummary, NOOP_SPAN, NOOP_SPAN_PERMALINK, NoopSpan, type ObjectMetadata, Project, ProjectNameIdMap, type PromiseUnless, Prompt, PromptBuilder, type PromptContents, type PromptDefinition, type PromptDefinitionWithTools, type PromptOpts, type PromptRowWithId, ReadonlyAttachment, ReadonlyExperiment, Reporter, type ReporterBody, type ScoreSummary, ScorerBuilder, type ScorerOpts, type SerializedBraintrustState, type SetCurrentArg, type Span$1 as Span, type SpanContext, SpanImpl, type StartSpanArgs, TestBackgroundLogger, ToolBuilder, type WithTransactionId, X_CACHED_HEADER, _exportsForTestingOnly, _internalGetGlobalState, _internalSetInitialState, braintrustStreamChunkSchema, buildLocalSummary, createFinalValuePassThroughStream, currentExperiment, currentLogger, currentSpan, braintrust as default, defaultErrorScoreHandler, deserializePlainStringAsJSON, devNullWritableStream, evaluatorDefinitionSchema, evaluatorDefinitionsSchema, flush, getSpanParentObject, graphFramework as graph, init, initDataset, initExperiment, initFunction, initLogger, invoke, loadPrompt, log, logError, login, loginToState, newId, parseCachedHeader, permalink, projects, promptContentsSchema, promptDefinitionSchema, promptDefinitionToPromptData, promptDefinitionWithToolsSchema, renderMessage, renderPromptParams, reportFailures, runEvaluator, setFetch, spanComponentsToObjectId, startSpan, summarize, traceable, traced, updateSpan, withCurrent, withDataset, withExperiment, withLogger, withParent, wrapAISDKModel, wrapAnthropic, wrapOpenAI, wrapOpenAIv4, wrapTraced };
|
package/dist/index.d.ts
CHANGED
|
@@ -1691,6 +1691,7 @@ interface DatasetSummary {
|
|
|
1691
1691
|
datasetUrl: string;
|
|
1692
1692
|
dataSummary: DataSummary | undefined;
|
|
1693
1693
|
}
|
|
1694
|
+
declare function setInitialTestState(): void;
|
|
1694
1695
|
declare function simulateLoginForTests(): Promise<BraintrustState>;
|
|
1695
1696
|
declare function simulateLogoutForTests(): BraintrustState;
|
|
1696
1697
|
declare const _exportsForTestingOnly: {
|
|
@@ -1700,6 +1701,7 @@ declare const _exportsForTestingOnly: {
|
|
|
1700
1701
|
clearTestBackgroundLogger: typeof clearTestBackgroundLogger;
|
|
1701
1702
|
simulateLoginForTests: typeof simulateLoginForTests;
|
|
1702
1703
|
simulateLogoutForTests: typeof simulateLogoutForTests;
|
|
1704
|
+
setInitialTestState: typeof setInitialTestState;
|
|
1703
1705
|
};
|
|
1704
1706
|
|
|
1705
1707
|
declare const braintrustStreamChunkSchema: z.ZodUnion<[z.ZodObject<{
|
|
@@ -7134,7 +7136,7 @@ declare global {
|
|
|
7134
7136
|
* not configured, nothing will be traced. If this is not an `OpenAI` object, this function is
|
|
7135
7137
|
* a no-op.
|
|
7136
7138
|
*
|
|
7137
|
-
* Currently, this
|
|
7139
|
+
* Currently, this supports both the `v4` and `v5` API.
|
|
7138
7140
|
*
|
|
7139
7141
|
* @param openai
|
|
7140
7142
|
* @returns The wrapped `OpenAI` object.
|
|
@@ -7145,6 +7147,50 @@ declare const LEGACY_CACHED_HEADER = "x-cached";
|
|
|
7145
7147
|
declare const X_CACHED_HEADER = "x-bt-cached";
|
|
7146
7148
|
declare function parseCachedHeader(value: string | null | undefined): number | undefined;
|
|
7147
7149
|
|
|
7150
|
+
interface LanguageModelV2Middleware<TModel = any, TCallOptions = any> {
|
|
7151
|
+
wrapGenerate?: (options: {
|
|
7152
|
+
doGenerate: () => any;
|
|
7153
|
+
doStream: () => any;
|
|
7154
|
+
params: TCallOptions;
|
|
7155
|
+
model: TModel;
|
|
7156
|
+
}) => Promise<any>;
|
|
7157
|
+
wrapStream?: (options: {
|
|
7158
|
+
doGenerate: () => any;
|
|
7159
|
+
doStream: () => any;
|
|
7160
|
+
params: TCallOptions;
|
|
7161
|
+
model: TModel;
|
|
7162
|
+
}) => Promise<any>;
|
|
7163
|
+
}
|
|
7164
|
+
/**
|
|
7165
|
+
* Configuration options for the AI SDK middleware
|
|
7166
|
+
*/
|
|
7167
|
+
interface MiddlewareConfig {
|
|
7168
|
+
/** Enable debug logging */
|
|
7169
|
+
debug?: boolean;
|
|
7170
|
+
/** Name identifier for the middleware instance */
|
|
7171
|
+
name?: string;
|
|
7172
|
+
}
|
|
7173
|
+
/**
|
|
7174
|
+
* Creates a Braintrust middleware for AI SDK v2 that automatically traces
|
|
7175
|
+
* generateText and streamText calls with comprehensive metadata and metrics.
|
|
7176
|
+
*
|
|
7177
|
+
* @param config - Configuration options for the middleware
|
|
7178
|
+
* @returns A middleware object compatible with AI SDK v2's wrapLanguageModel
|
|
7179
|
+
*
|
|
7180
|
+
* @example
|
|
7181
|
+
* ```typescript
|
|
7182
|
+
* import { wrapLanguageModel } from "ai";
|
|
7183
|
+
* import { openai } from "@ai-sdk/openai";
|
|
7184
|
+
* import { BraintrustMiddleware } from "braintrust";
|
|
7185
|
+
*
|
|
7186
|
+
* const model = wrapLanguageModel({
|
|
7187
|
+
* model: openai("gpt-4"),
|
|
7188
|
+
* middleware: BraintrustMiddleware({ debug: true, name: "MyMiddleware" })
|
|
7189
|
+
* });
|
|
7190
|
+
* ```
|
|
7191
|
+
*/
|
|
7192
|
+
declare function BraintrustMiddleware(config?: MiddlewareConfig): LanguageModelV2Middleware<any, any>;
|
|
7193
|
+
|
|
7148
7194
|
/**
|
|
7149
7195
|
* Wrap an ai-sdk model (created with `.chat()`, `.completion()`, etc.) to add tracing. If Braintrust is
|
|
7150
7196
|
* not configured, this is a no-op
|
|
@@ -7406,6 +7452,7 @@ declare const braintrust_BaseExperiment: typeof BaseExperiment;
|
|
|
7406
7452
|
type braintrust_BaseMetadata = BaseMetadata;
|
|
7407
7453
|
type braintrust_BraintrustExporter = BraintrustExporter;
|
|
7408
7454
|
declare const braintrust_BraintrustExporter: typeof BraintrustExporter;
|
|
7455
|
+
declare const braintrust_BraintrustMiddleware: typeof BraintrustMiddleware;
|
|
7409
7456
|
type braintrust_BraintrustSpanProcessor = BraintrustSpanProcessor;
|
|
7410
7457
|
declare const braintrust_BraintrustSpanProcessor: typeof BraintrustSpanProcessor;
|
|
7411
7458
|
type braintrust_BraintrustState = BraintrustState;
|
|
@@ -7564,7 +7611,7 @@ declare const braintrust_wrapOpenAI: typeof wrapOpenAI;
|
|
|
7564
7611
|
declare const braintrust_wrapOpenAIv4: typeof wrapOpenAIv4;
|
|
7565
7612
|
declare const braintrust_wrapTraced: typeof wrapTraced;
|
|
7566
7613
|
declare namespace braintrust {
|
|
7567
|
-
export { braintrust_AISpanProcessor as AISpanProcessor, type braintrust_AnyDataset as AnyDataset, braintrust_Attachment as Attachment, type braintrust_AttachmentParams as AttachmentParams, type braintrust_BackgroundLoggerOpts as BackgroundLoggerOpts, braintrust_BaseAttachment as BaseAttachment, braintrust_BaseExperiment as BaseExperiment, type braintrust_BaseMetadata as BaseMetadata, braintrust_BraintrustExporter as BraintrustExporter, braintrust_BraintrustSpanProcessor as BraintrustSpanProcessor, braintrust_BraintrustState as BraintrustState, braintrust_BraintrustStream as BraintrustStream, type braintrust_BraintrustStreamChunk as BraintrustStreamChunk, type braintrust_ChatPrompt as ChatPrompt, braintrust_CodeFunction as CodeFunction, type braintrust_CodeOpts as CodeOpts, braintrust_CodePrompt as CodePrompt, type braintrust_CompiledPrompt as CompiledPrompt, type braintrust_CompiledPromptParams as CompiledPromptParams, type braintrust_CompletionPrompt as CompletionPrompt, type braintrust_CreateProjectOpts as CreateProjectOpts, type braintrust_DataSummary as DataSummary, braintrust_Dataset as Dataset, type braintrust_DatasetSummary as DatasetSummary, type braintrust_DefaultMetadataType as DefaultMetadataType, type braintrust_DefaultPromptArgs as DefaultPromptArgs, braintrust_ERR_PERMALINK as ERR_PERMALINK, type braintrust_EndSpanArgs as EndSpanArgs, braintrust_Eval as Eval, type braintrust_EvalCase as EvalCase, type braintrust_EvalHooks as EvalHooks, type braintrust_EvalResult as EvalResult, braintrust_EvalResultWithSummary as EvalResultWithSummary, type braintrust_EvalScorer as EvalScorer, type braintrust_EvalScorerArgs as EvalScorerArgs, type braintrust_EvalTask as EvalTask, type braintrust_Evaluator as Evaluator, type braintrust_EvaluatorDef as EvaluatorDef, type braintrust_EvaluatorFile as EvaluatorFile, braintrust_Experiment as Experiment, type braintrust_ExperimentSummary as ExperimentSummary, type braintrust_Exportable as Exportable, braintrust_ExternalAttachment as ExternalAttachment, type braintrust_ExternalAttachmentParams as ExternalAttachmentParams, braintrust_FailedHTTPResponse as FailedHTTPResponse, type braintrust_FullInitOptions as FullInitOptions, type braintrust_FullLoginOptions as FullLoginOptions, type braintrust_FunctionEvent as FunctionEvent, braintrust_INTERNAL_BTQL_LIMIT as INTERNAL_BTQL_LIMIT, type braintrust_InitOptions as InitOptions, type braintrust_InvokeFunctionArgs as InvokeFunctionArgs, type braintrust_InvokeReturn as InvokeReturn, braintrust_LEGACY_CACHED_HEADER as LEGACY_CACHED_HEADER, braintrust_LazyValue as LazyValue, type braintrust_LogOptions as LogOptions, braintrust_Logger as Logger, type braintrust_LoginOptions as LoginOptions, type braintrust_MetricSummary as MetricSummary, braintrust_NOOP_SPAN as NOOP_SPAN, braintrust_NOOP_SPAN_PERMALINK as NOOP_SPAN_PERMALINK, braintrust_NoopSpan as NoopSpan, type braintrust_ObjectMetadata as ObjectMetadata, braintrust_Project as Project, braintrust_ProjectNameIdMap as ProjectNameIdMap, type braintrust_PromiseUnless as PromiseUnless, braintrust_Prompt as Prompt, braintrust_PromptBuilder as PromptBuilder, type braintrust_PromptContents as PromptContents, type braintrust_PromptDefinition as PromptDefinition, type braintrust_PromptDefinitionWithTools as PromptDefinitionWithTools, type braintrust_PromptOpts as PromptOpts, type braintrust_PromptRowWithId as PromptRowWithId, braintrust_ReadonlyAttachment as ReadonlyAttachment, braintrust_ReadonlyExperiment as ReadonlyExperiment, braintrust_Reporter as Reporter, type braintrust_ReporterBody as ReporterBody, type braintrust_ScoreSummary as ScoreSummary, braintrust_ScorerBuilder as ScorerBuilder, type braintrust_ScorerOpts as ScorerOpts, type braintrust_SerializedBraintrustState as SerializedBraintrustState, type braintrust_SetCurrentArg as SetCurrentArg, type Span$1 as Span, type braintrust_SpanContext as SpanContext, braintrust_SpanImpl as SpanImpl, type braintrust_StartSpanArgs as StartSpanArgs, braintrust_TestBackgroundLogger as TestBackgroundLogger, braintrust_ToolBuilder as ToolBuilder, type braintrust_WithTransactionId as WithTransactionId, braintrust_X_CACHED_HEADER as X_CACHED_HEADER, braintrust__exportsForTestingOnly as _exportsForTestingOnly, braintrust__internalGetGlobalState as _internalGetGlobalState, braintrust__internalSetInitialState as _internalSetInitialState, braintrust_braintrustStreamChunkSchema as braintrustStreamChunkSchema, braintrust_buildLocalSummary as buildLocalSummary, braintrust_createFinalValuePassThroughStream as createFinalValuePassThroughStream, braintrust_currentExperiment as currentExperiment, braintrust_currentLogger as currentLogger, braintrust_currentSpan as currentSpan, braintrust_defaultErrorScoreHandler as defaultErrorScoreHandler, braintrust_deserializePlainStringAsJSON as deserializePlainStringAsJSON, braintrust_devNullWritableStream as devNullWritableStream, braintrust_flush as flush, braintrust_getSpanParentObject as getSpanParentObject, graphFramework as graph, braintrust_init as init, braintrust_initDataset as initDataset, braintrust_initExperiment as initExperiment, braintrust_initFunction as initFunction, braintrust_initLogger as initLogger, braintrust_invoke as invoke, braintrust_loadPrompt as loadPrompt, braintrust_log as log, braintrust_logError as logError, braintrust_login as login, braintrust_loginToState as loginToState, braintrust_newId as newId, braintrust_parseCachedHeader as parseCachedHeader, braintrust_permalink as permalink, braintrust_projects as projects, braintrust_promptContentsSchema as promptContentsSchema, braintrust_promptDefinitionSchema as promptDefinitionSchema, braintrust_promptDefinitionToPromptData as promptDefinitionToPromptData, braintrust_promptDefinitionWithToolsSchema as promptDefinitionWithToolsSchema, braintrust_renderMessage as renderMessage, braintrust_renderPromptParams as renderPromptParams, braintrust_reportFailures as reportFailures, braintrust_runEvaluator as runEvaluator, braintrust_setFetch as setFetch, braintrust_spanComponentsToObjectId as spanComponentsToObjectId, braintrust_startSpan as startSpan, braintrust_summarize as summarize, braintrust_toolFunctionDefinitionSchema as toolFunctionDefinitionSchema, braintrust_traceable as traceable, braintrust_traced as traced, braintrust_updateSpan as updateSpan, braintrust_withCurrent as withCurrent, braintrust_withDataset as withDataset, braintrust_withExperiment as withExperiment, braintrust_withLogger as withLogger, braintrust_withParent as withParent, braintrust_wrapAISDKModel as wrapAISDKModel, braintrust_wrapAnthropic as wrapAnthropic, braintrust_wrapOpenAI as wrapOpenAI, braintrust_wrapOpenAIv4 as wrapOpenAIv4, braintrust_wrapTraced as wrapTraced };
|
|
7614
|
+
export { braintrust_AISpanProcessor as AISpanProcessor, type braintrust_AnyDataset as AnyDataset, braintrust_Attachment as Attachment, type braintrust_AttachmentParams as AttachmentParams, type braintrust_BackgroundLoggerOpts as BackgroundLoggerOpts, braintrust_BaseAttachment as BaseAttachment, braintrust_BaseExperiment as BaseExperiment, type braintrust_BaseMetadata as BaseMetadata, braintrust_BraintrustExporter as BraintrustExporter, braintrust_BraintrustMiddleware as BraintrustMiddleware, braintrust_BraintrustSpanProcessor as BraintrustSpanProcessor, braintrust_BraintrustState as BraintrustState, braintrust_BraintrustStream as BraintrustStream, type braintrust_BraintrustStreamChunk as BraintrustStreamChunk, type braintrust_ChatPrompt as ChatPrompt, braintrust_CodeFunction as CodeFunction, type braintrust_CodeOpts as CodeOpts, braintrust_CodePrompt as CodePrompt, type braintrust_CompiledPrompt as CompiledPrompt, type braintrust_CompiledPromptParams as CompiledPromptParams, type braintrust_CompletionPrompt as CompletionPrompt, type braintrust_CreateProjectOpts as CreateProjectOpts, type braintrust_DataSummary as DataSummary, braintrust_Dataset as Dataset, type braintrust_DatasetSummary as DatasetSummary, type braintrust_DefaultMetadataType as DefaultMetadataType, type braintrust_DefaultPromptArgs as DefaultPromptArgs, braintrust_ERR_PERMALINK as ERR_PERMALINK, type braintrust_EndSpanArgs as EndSpanArgs, braintrust_Eval as Eval, type braintrust_EvalCase as EvalCase, type braintrust_EvalHooks as EvalHooks, type braintrust_EvalResult as EvalResult, braintrust_EvalResultWithSummary as EvalResultWithSummary, type braintrust_EvalScorer as EvalScorer, type braintrust_EvalScorerArgs as EvalScorerArgs, type braintrust_EvalTask as EvalTask, type braintrust_Evaluator as Evaluator, type braintrust_EvaluatorDef as EvaluatorDef, type braintrust_EvaluatorFile as EvaluatorFile, braintrust_Experiment as Experiment, type braintrust_ExperimentSummary as ExperimentSummary, type braintrust_Exportable as Exportable, braintrust_ExternalAttachment as ExternalAttachment, type braintrust_ExternalAttachmentParams as ExternalAttachmentParams, braintrust_FailedHTTPResponse as FailedHTTPResponse, type braintrust_FullInitOptions as FullInitOptions, type braintrust_FullLoginOptions as FullLoginOptions, type braintrust_FunctionEvent as FunctionEvent, braintrust_INTERNAL_BTQL_LIMIT as INTERNAL_BTQL_LIMIT, type braintrust_InitOptions as InitOptions, type braintrust_InvokeFunctionArgs as InvokeFunctionArgs, type braintrust_InvokeReturn as InvokeReturn, braintrust_LEGACY_CACHED_HEADER as LEGACY_CACHED_HEADER, braintrust_LazyValue as LazyValue, type braintrust_LogOptions as LogOptions, braintrust_Logger as Logger, type braintrust_LoginOptions as LoginOptions, type braintrust_MetricSummary as MetricSummary, braintrust_NOOP_SPAN as NOOP_SPAN, braintrust_NOOP_SPAN_PERMALINK as NOOP_SPAN_PERMALINK, braintrust_NoopSpan as NoopSpan, type braintrust_ObjectMetadata as ObjectMetadata, braintrust_Project as Project, braintrust_ProjectNameIdMap as ProjectNameIdMap, type braintrust_PromiseUnless as PromiseUnless, braintrust_Prompt as Prompt, braintrust_PromptBuilder as PromptBuilder, type braintrust_PromptContents as PromptContents, type braintrust_PromptDefinition as PromptDefinition, type braintrust_PromptDefinitionWithTools as PromptDefinitionWithTools, type braintrust_PromptOpts as PromptOpts, type braintrust_PromptRowWithId as PromptRowWithId, braintrust_ReadonlyAttachment as ReadonlyAttachment, braintrust_ReadonlyExperiment as ReadonlyExperiment, braintrust_Reporter as Reporter, type braintrust_ReporterBody as ReporterBody, type braintrust_ScoreSummary as ScoreSummary, braintrust_ScorerBuilder as ScorerBuilder, type braintrust_ScorerOpts as ScorerOpts, type braintrust_SerializedBraintrustState as SerializedBraintrustState, type braintrust_SetCurrentArg as SetCurrentArg, type Span$1 as Span, type braintrust_SpanContext as SpanContext, braintrust_SpanImpl as SpanImpl, type braintrust_StartSpanArgs as StartSpanArgs, braintrust_TestBackgroundLogger as TestBackgroundLogger, braintrust_ToolBuilder as ToolBuilder, type braintrust_WithTransactionId as WithTransactionId, braintrust_X_CACHED_HEADER as X_CACHED_HEADER, braintrust__exportsForTestingOnly as _exportsForTestingOnly, braintrust__internalGetGlobalState as _internalGetGlobalState, braintrust__internalSetInitialState as _internalSetInitialState, braintrust_braintrustStreamChunkSchema as braintrustStreamChunkSchema, braintrust_buildLocalSummary as buildLocalSummary, braintrust_createFinalValuePassThroughStream as createFinalValuePassThroughStream, braintrust_currentExperiment as currentExperiment, braintrust_currentLogger as currentLogger, braintrust_currentSpan as currentSpan, braintrust_defaultErrorScoreHandler as defaultErrorScoreHandler, braintrust_deserializePlainStringAsJSON as deserializePlainStringAsJSON, braintrust_devNullWritableStream as devNullWritableStream, braintrust_flush as flush, braintrust_getSpanParentObject as getSpanParentObject, graphFramework as graph, braintrust_init as init, braintrust_initDataset as initDataset, braintrust_initExperiment as initExperiment, braintrust_initFunction as initFunction, braintrust_initLogger as initLogger, braintrust_invoke as invoke, braintrust_loadPrompt as loadPrompt, braintrust_log as log, braintrust_logError as logError, braintrust_login as login, braintrust_loginToState as loginToState, braintrust_newId as newId, braintrust_parseCachedHeader as parseCachedHeader, braintrust_permalink as permalink, braintrust_projects as projects, braintrust_promptContentsSchema as promptContentsSchema, braintrust_promptDefinitionSchema as promptDefinitionSchema, braintrust_promptDefinitionToPromptData as promptDefinitionToPromptData, braintrust_promptDefinitionWithToolsSchema as promptDefinitionWithToolsSchema, braintrust_renderMessage as renderMessage, braintrust_renderPromptParams as renderPromptParams, braintrust_reportFailures as reportFailures, braintrust_runEvaluator as runEvaluator, braintrust_setFetch as setFetch, braintrust_spanComponentsToObjectId as spanComponentsToObjectId, braintrust_startSpan as startSpan, braintrust_summarize as summarize, braintrust_toolFunctionDefinitionSchema as toolFunctionDefinitionSchema, braintrust_traceable as traceable, braintrust_traced as traced, braintrust_updateSpan as updateSpan, braintrust_withCurrent as withCurrent, braintrust_withDataset as withDataset, braintrust_withExperiment as withExperiment, braintrust_withLogger as withLogger, braintrust_withParent as withParent, braintrust_wrapAISDKModel as wrapAISDKModel, braintrust_wrapAnthropic as wrapAnthropic, braintrust_wrapOpenAI as wrapOpenAI, braintrust_wrapOpenAIv4 as wrapOpenAIv4, braintrust_wrapTraced as wrapTraced };
|
|
7568
7615
|
}
|
|
7569
7616
|
|
|
7570
7617
|
type EvaluatorManifest = Record<string, EvaluatorDef<unknown, unknown, unknown, BaseMetadata>>;
|
|
@@ -14792,4 +14839,4 @@ type EvaluatorDefinitions = z.infer<typeof evaluatorDefinitionsSchema>;
|
|
|
14792
14839
|
* @module braintrust
|
|
14793
14840
|
*/
|
|
14794
14841
|
|
|
14795
|
-
export { AISpanProcessor, type AnyDataset, Attachment, type AttachmentParams, type BackgroundLoggerOpts, BaseAttachment, BaseExperiment, type BaseMetadata, BraintrustExporter, BraintrustSpanProcessor, BraintrustState, BraintrustStream, type BraintrustStreamChunk, type ChatPrompt, CodeFunction, type CodeOpts, CodePrompt, type CompiledPrompt, type CompiledPromptParams, type CompletionPrompt, type CreateProjectOpts, type DataSummary, Dataset, type DatasetSummary, type DefaultMetadataType, type DefaultPromptArgs, ERR_PERMALINK, type EndSpanArgs, Eval, type EvalCase, type EvalHooks, type EvalParameterSerializedSchema, type EvalParameters, type EvalResult, EvalResultWithSummary, type EvalScorer, type EvalScorerArgs, type EvalTask, type Evaluator, type EvaluatorDef, type EvaluatorDefinition, type EvaluatorDefinitions, type EvaluatorFile, type EvaluatorManifest, Experiment, type ExperimentSummary, type Exportable, ExternalAttachment, type ExternalAttachmentParams, FailedHTTPResponse, type FullInitOptions, type FullLoginOptions, type FunctionEvent, INTERNAL_BTQL_LIMIT, type InitOptions, type InvokeFunctionArgs, type InvokeReturn, LEGACY_CACHED_HEADER, LazyValue, type LogOptions, Logger, type LoginOptions, type MetricSummary, NOOP_SPAN, NOOP_SPAN_PERMALINK, NoopSpan, type ObjectMetadata, Project, ProjectNameIdMap, type PromiseUnless, Prompt, PromptBuilder, type PromptContents, type PromptDefinition, type PromptDefinitionWithTools, type PromptOpts, type PromptRowWithId, ReadonlyAttachment, ReadonlyExperiment, Reporter, type ReporterBody, type ScoreSummary, ScorerBuilder, type ScorerOpts, type SerializedBraintrustState, type SetCurrentArg, type Span$1 as Span, type SpanContext, SpanImpl, type StartSpanArgs, TestBackgroundLogger, ToolBuilder, type WithTransactionId, X_CACHED_HEADER, _exportsForTestingOnly, _internalGetGlobalState, _internalSetInitialState, braintrustStreamChunkSchema, buildLocalSummary, createFinalValuePassThroughStream, currentExperiment, currentLogger, currentSpan, braintrust as default, defaultErrorScoreHandler, deserializePlainStringAsJSON, devNullWritableStream, evaluatorDefinitionSchema, evaluatorDefinitionsSchema, flush, getSpanParentObject, graphFramework as graph, init, initDataset, initExperiment, initFunction, initLogger, invoke, loadPrompt, log, logError, login, loginToState, newId, parseCachedHeader, permalink, projects, promptContentsSchema, promptDefinitionSchema, promptDefinitionToPromptData, promptDefinitionWithToolsSchema, renderMessage, renderPromptParams, reportFailures, runEvaluator, setFetch, spanComponentsToObjectId, startSpan, summarize, traceable, traced, updateSpan, withCurrent, withDataset, withExperiment, withLogger, withParent, wrapAISDKModel, wrapAnthropic, wrapOpenAI, wrapOpenAIv4, wrapTraced };
|
|
14842
|
+
export { AISpanProcessor, type AnyDataset, Attachment, type AttachmentParams, type BackgroundLoggerOpts, BaseAttachment, BaseExperiment, type BaseMetadata, BraintrustExporter, BraintrustMiddleware, BraintrustSpanProcessor, BraintrustState, BraintrustStream, type BraintrustStreamChunk, type ChatPrompt, CodeFunction, type CodeOpts, CodePrompt, type CompiledPrompt, type CompiledPromptParams, type CompletionPrompt, type CreateProjectOpts, type DataSummary, Dataset, type DatasetSummary, type DefaultMetadataType, type DefaultPromptArgs, ERR_PERMALINK, type EndSpanArgs, Eval, type EvalCase, type EvalHooks, type EvalParameterSerializedSchema, type EvalParameters, type EvalResult, EvalResultWithSummary, type EvalScorer, type EvalScorerArgs, type EvalTask, type Evaluator, type EvaluatorDef, type EvaluatorDefinition, type EvaluatorDefinitions, type EvaluatorFile, type EvaluatorManifest, Experiment, type ExperimentSummary, type Exportable, ExternalAttachment, type ExternalAttachmentParams, FailedHTTPResponse, type FullInitOptions, type FullLoginOptions, type FunctionEvent, INTERNAL_BTQL_LIMIT, type InitOptions, type InvokeFunctionArgs, type InvokeReturn, LEGACY_CACHED_HEADER, LazyValue, type LogOptions, Logger, type LoginOptions, type MetricSummary, NOOP_SPAN, NOOP_SPAN_PERMALINK, NoopSpan, type ObjectMetadata, Project, ProjectNameIdMap, type PromiseUnless, Prompt, PromptBuilder, type PromptContents, type PromptDefinition, type PromptDefinitionWithTools, type PromptOpts, type PromptRowWithId, ReadonlyAttachment, ReadonlyExperiment, Reporter, type ReporterBody, type ScoreSummary, ScorerBuilder, type ScorerOpts, type SerializedBraintrustState, type SetCurrentArg, type Span$1 as Span, type SpanContext, SpanImpl, type StartSpanArgs, TestBackgroundLogger, ToolBuilder, type WithTransactionId, X_CACHED_HEADER, _exportsForTestingOnly, _internalGetGlobalState, _internalSetInitialState, braintrustStreamChunkSchema, buildLocalSummary, createFinalValuePassThroughStream, currentExperiment, currentLogger, currentSpan, braintrust as default, defaultErrorScoreHandler, deserializePlainStringAsJSON, devNullWritableStream, evaluatorDefinitionSchema, evaluatorDefinitionsSchema, flush, getSpanParentObject, graphFramework as graph, init, initDataset, initExperiment, initFunction, initLogger, invoke, loadPrompt, log, logError, login, loginToState, newId, parseCachedHeader, permalink, projects, promptContentsSchema, promptDefinitionSchema, promptDefinitionToPromptData, promptDefinitionWithToolsSchema, renderMessage, renderPromptParams, reportFailures, runEvaluator, setFetch, spanComponentsToObjectId, startSpan, summarize, traceable, traced, updateSpan, withCurrent, withDataset, withExperiment, withLogger, withParent, wrapAISDKModel, wrapAnthropic, wrapOpenAI, wrapOpenAIv4, wrapTraced };
|
package/dist/index.js
CHANGED
|
@@ -7979,6 +7979,11 @@ var Prompt = (_class23 = class _Prompt {
|
|
|
7979
7979
|
}
|
|
7980
7980
|
}, _class23);
|
|
7981
7981
|
var TEST_API_KEY = "___TEST_API_KEY__THIS_IS_NOT_REAL___";
|
|
7982
|
+
function setInitialTestState() {
|
|
7983
|
+
if (!_internalGetGlobalState()) {
|
|
7984
|
+
_internalSetInitialState();
|
|
7985
|
+
}
|
|
7986
|
+
}
|
|
7982
7987
|
async function simulateLoginForTests() {
|
|
7983
7988
|
return await login({
|
|
7984
7989
|
apiKey: TEST_API_KEY,
|
|
@@ -7996,7 +8001,8 @@ var _exportsForTestingOnly = {
|
|
|
7996
8001
|
useTestBackgroundLogger,
|
|
7997
8002
|
clearTestBackgroundLogger,
|
|
7998
8003
|
simulateLoginForTests,
|
|
7999
|
-
simulateLogoutForTests
|
|
8004
|
+
simulateLogoutForTests,
|
|
8005
|
+
setInitialTestState
|
|
8000
8006
|
};
|
|
8001
8007
|
|
|
8002
8008
|
// src/node.ts
|
|
@@ -8035,6 +8041,7 @@ _chunkCDBUTZMHjs.__export.call(void 0, exports_node_exports, {
|
|
|
8035
8041
|
BaseAttachment: () => BaseAttachment,
|
|
8036
8042
|
BaseExperiment: () => BaseExperiment,
|
|
8037
8043
|
BraintrustExporter: () => BraintrustExporter,
|
|
8044
|
+
BraintrustMiddleware: () => BraintrustMiddleware,
|
|
8038
8045
|
BraintrustSpanProcessor: () => BraintrustSpanProcessor,
|
|
8039
8046
|
BraintrustState: () => BraintrustState,
|
|
8040
8047
|
BraintrustStream: () => BraintrustStream,
|
|
@@ -10841,10 +10848,17 @@ function wrapOpenAI(openai) {
|
|
|
10841
10848
|
}
|
|
10842
10849
|
globalThis.__inherited_braintrust_wrap_openai = wrapOpenAI;
|
|
10843
10850
|
function wrapOpenAIv4(openai) {
|
|
10844
|
-
const completionProxy =
|
|
10845
|
-
|
|
10846
|
-
|
|
10847
|
-
|
|
10851
|
+
const completionProxy = new Proxy(openai.chat.completions, {
|
|
10852
|
+
get(target, name, receiver) {
|
|
10853
|
+
const baseVal = Reflect.get(target, name, receiver);
|
|
10854
|
+
if (name === "create") {
|
|
10855
|
+
return wrapChatCompletion(baseVal.bind(target));
|
|
10856
|
+
} else if (name === "parse") {
|
|
10857
|
+
return wrapBetaChatCompletionParse(baseVal.bind(target));
|
|
10858
|
+
}
|
|
10859
|
+
return baseVal;
|
|
10860
|
+
}
|
|
10861
|
+
});
|
|
10848
10862
|
const chatProxy = new Proxy(openai.chat, {
|
|
10849
10863
|
get(target, name, receiver) {
|
|
10850
10864
|
if (name === "completions") {
|
|
@@ -11220,10 +11234,221 @@ var WrapperStream = class {
|
|
|
11220
11234
|
}
|
|
11221
11235
|
};
|
|
11222
11236
|
|
|
11223
|
-
// src/wrappers/ai-sdk.ts
|
|
11237
|
+
// src/wrappers/ai-sdk-v2.ts
|
|
11238
|
+
|
|
11239
|
+
function detectProviderFromResult(result) {
|
|
11240
|
+
if (!_optionalChain([result, 'optionalAccess', _176 => _176.providerMetadata])) {
|
|
11241
|
+
return void 0;
|
|
11242
|
+
}
|
|
11243
|
+
const keys = Object.keys(result.providerMetadata);
|
|
11244
|
+
return _optionalChain([keys, 'optionalAccess', _177 => _177.at, 'call', _178 => _178(0)]);
|
|
11245
|
+
}
|
|
11246
|
+
function extractModelFromResult(result) {
|
|
11247
|
+
if (_optionalChain([result, 'optionalAccess', _179 => _179.response, 'optionalAccess', _180 => _180.modelId])) {
|
|
11248
|
+
return result.response.modelId;
|
|
11249
|
+
}
|
|
11250
|
+
if (_optionalChain([result, 'optionalAccess', _181 => _181.request, 'optionalAccess', _182 => _182.body, 'optionalAccess', _183 => _183.model])) {
|
|
11251
|
+
return result.request.body.model;
|
|
11252
|
+
}
|
|
11253
|
+
return void 0;
|
|
11254
|
+
}
|
|
11255
|
+
function camelToSnake(str) {
|
|
11256
|
+
return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
|
|
11257
|
+
}
|
|
11258
|
+
function extractModelParameters(params) {
|
|
11259
|
+
const modelParams = {};
|
|
11260
|
+
const excludeKeys = /* @__PURE__ */ new Set([
|
|
11261
|
+
"prompt",
|
|
11262
|
+
// Already captured as input
|
|
11263
|
+
"system",
|
|
11264
|
+
// Already captured as input
|
|
11265
|
+
"messages",
|
|
11266
|
+
// Already captured as input
|
|
11267
|
+
"model",
|
|
11268
|
+
// Already captured in metadata.model
|
|
11269
|
+
"providerOptions"
|
|
11270
|
+
// Internal AI SDK configuration
|
|
11271
|
+
]);
|
|
11272
|
+
for (const [key, value] of Object.entries(params)) {
|
|
11273
|
+
if (value !== void 0 && !excludeKeys.has(key)) {
|
|
11274
|
+
const snakeKey = camelToSnake(key);
|
|
11275
|
+
modelParams[snakeKey] = value;
|
|
11276
|
+
}
|
|
11277
|
+
}
|
|
11278
|
+
return modelParams;
|
|
11279
|
+
}
|
|
11280
|
+
function getNumberProperty(obj, key) {
|
|
11281
|
+
if (!obj || typeof obj !== "object" || !(key in obj)) {
|
|
11282
|
+
return void 0;
|
|
11283
|
+
}
|
|
11284
|
+
const value = Reflect.get(obj, key);
|
|
11285
|
+
return typeof value === "number" ? value : void 0;
|
|
11286
|
+
}
|
|
11287
|
+
function normalizeUsageMetrics(usage) {
|
|
11288
|
+
const metrics = {};
|
|
11289
|
+
const inputTokens = getNumberProperty(usage, "inputTokens");
|
|
11290
|
+
if (inputTokens !== void 0) {
|
|
11291
|
+
metrics.prompt_tokens = inputTokens;
|
|
11292
|
+
}
|
|
11293
|
+
const outputTokens = getNumberProperty(usage, "outputTokens");
|
|
11294
|
+
if (outputTokens !== void 0) {
|
|
11295
|
+
metrics.completion_tokens = outputTokens;
|
|
11296
|
+
}
|
|
11297
|
+
const totalTokens = getNumberProperty(usage, "totalTokens");
|
|
11298
|
+
if (totalTokens !== void 0) {
|
|
11299
|
+
metrics.tokens = totalTokens;
|
|
11300
|
+
}
|
|
11301
|
+
const reasoningTokens = getNumberProperty(usage, "reasoningTokens");
|
|
11302
|
+
if (reasoningTokens !== void 0) {
|
|
11303
|
+
metrics.completion_reasoning_tokens = reasoningTokens;
|
|
11304
|
+
}
|
|
11305
|
+
const cachedInputTokens = getNumberProperty(usage, "cachedInputTokens");
|
|
11306
|
+
if (cachedInputTokens !== void 0) {
|
|
11307
|
+
metrics.prompt_cached_tokens = cachedInputTokens;
|
|
11308
|
+
}
|
|
11309
|
+
return metrics;
|
|
11310
|
+
}
|
|
11311
|
+
function BraintrustMiddleware(config = {}) {
|
|
11312
|
+
return {
|
|
11313
|
+
wrapGenerate: async ({ doGenerate, params }) => {
|
|
11314
|
+
const spanArgs = {
|
|
11315
|
+
name: "ai-sdk.generateText",
|
|
11316
|
+
spanAttributes: {
|
|
11317
|
+
type: _core.SpanTypeAttribute.LLM
|
|
11318
|
+
},
|
|
11319
|
+
event: {
|
|
11320
|
+
input: params.prompt,
|
|
11321
|
+
metadata: {
|
|
11322
|
+
...extractModelParameters(params)
|
|
11323
|
+
}
|
|
11324
|
+
}
|
|
11325
|
+
};
|
|
11326
|
+
const span = startSpan(spanArgs);
|
|
11327
|
+
try {
|
|
11328
|
+
const result = await doGenerate();
|
|
11329
|
+
const metadata = {};
|
|
11330
|
+
const provider = detectProviderFromResult(result);
|
|
11331
|
+
if (provider !== void 0) {
|
|
11332
|
+
metadata.provider = provider;
|
|
11333
|
+
}
|
|
11334
|
+
if (result.finishReason !== void 0) {
|
|
11335
|
+
metadata.finish_reason = result.finishReason;
|
|
11336
|
+
}
|
|
11337
|
+
const model = extractModelFromResult(result);
|
|
11338
|
+
if (model !== void 0) {
|
|
11339
|
+
metadata.model = model;
|
|
11340
|
+
}
|
|
11341
|
+
span.log({
|
|
11342
|
+
output: result.content,
|
|
11343
|
+
metadata,
|
|
11344
|
+
metrics: normalizeUsageMetrics(result.usage)
|
|
11345
|
+
});
|
|
11346
|
+
return result;
|
|
11347
|
+
} catch (error2) {
|
|
11348
|
+
span.log({
|
|
11349
|
+
error: error2 instanceof Error ? error2.message : String(error2)
|
|
11350
|
+
});
|
|
11351
|
+
throw error2;
|
|
11352
|
+
} finally {
|
|
11353
|
+
span.end();
|
|
11354
|
+
}
|
|
11355
|
+
},
|
|
11356
|
+
wrapStream: async ({ doStream, params }) => {
|
|
11357
|
+
const spanArgs = {
|
|
11358
|
+
name: "ai-sdk.streamText",
|
|
11359
|
+
spanAttributes: {
|
|
11360
|
+
type: _core.SpanTypeAttribute.LLM
|
|
11361
|
+
},
|
|
11362
|
+
event: {
|
|
11363
|
+
input: params.prompt,
|
|
11364
|
+
metadata: {
|
|
11365
|
+
...extractModelParameters(params)
|
|
11366
|
+
}
|
|
11367
|
+
}
|
|
11368
|
+
};
|
|
11369
|
+
const span = startSpan(spanArgs);
|
|
11370
|
+
try {
|
|
11371
|
+
const { stream, ...rest } = await doStream();
|
|
11372
|
+
const textChunks = [];
|
|
11373
|
+
let finalUsage = {};
|
|
11374
|
+
let finalFinishReason = void 0;
|
|
11375
|
+
let providerMetadata = {};
|
|
11376
|
+
const transformStream = new TransformStream({
|
|
11377
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
11378
|
+
transform(chunk, controller) {
|
|
11379
|
+
try {
|
|
11380
|
+
if (chunk.type === "text-delta" && chunk.delta) {
|
|
11381
|
+
textChunks.push(chunk.delta);
|
|
11382
|
+
}
|
|
11383
|
+
if (chunk.type === "finish") {
|
|
11384
|
+
finalFinishReason = chunk.finishReason;
|
|
11385
|
+
finalUsage = chunk.usage || {};
|
|
11386
|
+
providerMetadata = chunk.providerMetadata || {};
|
|
11387
|
+
}
|
|
11388
|
+
controller.enqueue(chunk);
|
|
11389
|
+
} catch (error2) {
|
|
11390
|
+
span.log({
|
|
11391
|
+
error: error2 instanceof Error ? error2.message : String(error2)
|
|
11392
|
+
});
|
|
11393
|
+
span.end();
|
|
11394
|
+
controller.error(error2);
|
|
11395
|
+
}
|
|
11396
|
+
},
|
|
11397
|
+
flush() {
|
|
11398
|
+
try {
|
|
11399
|
+
const generatedText = textChunks.join("");
|
|
11400
|
+
const output = generatedText ? [{ type: "text", text: generatedText }] : [];
|
|
11401
|
+
const resultForDetection = {
|
|
11402
|
+
providerMetadata,
|
|
11403
|
+
response: rest.response,
|
|
11404
|
+
...rest
|
|
11405
|
+
};
|
|
11406
|
+
const metadata = {};
|
|
11407
|
+
const provider = detectProviderFromResult(resultForDetection);
|
|
11408
|
+
if (provider !== void 0) {
|
|
11409
|
+
metadata.provider = provider;
|
|
11410
|
+
}
|
|
11411
|
+
if (finalFinishReason !== void 0) {
|
|
11412
|
+
metadata.finish_reason = finalFinishReason;
|
|
11413
|
+
}
|
|
11414
|
+
const model = extractModelFromResult(resultForDetection);
|
|
11415
|
+
if (model !== void 0) {
|
|
11416
|
+
metadata.model = model;
|
|
11417
|
+
}
|
|
11418
|
+
span.log({
|
|
11419
|
+
output,
|
|
11420
|
+
metadata,
|
|
11421
|
+
metrics: normalizeUsageMetrics(finalUsage)
|
|
11422
|
+
});
|
|
11423
|
+
span.end();
|
|
11424
|
+
} catch (error2) {
|
|
11425
|
+
span.log({
|
|
11426
|
+
error: error2 instanceof Error ? error2.message : String(error2)
|
|
11427
|
+
});
|
|
11428
|
+
span.end();
|
|
11429
|
+
throw error2;
|
|
11430
|
+
}
|
|
11431
|
+
}
|
|
11432
|
+
});
|
|
11433
|
+
return {
|
|
11434
|
+
stream: stream.pipeThrough(transformStream),
|
|
11435
|
+
...rest
|
|
11436
|
+
};
|
|
11437
|
+
} catch (error2) {
|
|
11438
|
+
span.log({
|
|
11439
|
+
error: error2 instanceof Error ? error2.message : String(error2)
|
|
11440
|
+
});
|
|
11441
|
+
span.end();
|
|
11442
|
+
throw error2;
|
|
11443
|
+
}
|
|
11444
|
+
}
|
|
11445
|
+
};
|
|
11446
|
+
}
|
|
11447
|
+
|
|
11448
|
+
// src/wrappers/ai-sdk-v1.ts
|
|
11224
11449
|
function wrapAISDKModel(model) {
|
|
11225
11450
|
const m = model;
|
|
11226
|
-
if (_optionalChain([m, 'optionalAccess',
|
|
11451
|
+
if (_optionalChain([m, 'optionalAccess', _184 => _184.specificationVersion]) === "v1" && typeof _optionalChain([m, 'optionalAccess', _185 => _185.provider]) === "string" && typeof _optionalChain([m, 'optionalAccess', _186 => _186.modelId]) === "string") {
|
|
11227
11452
|
return new BraintrustLanguageModelWrapper(m);
|
|
11228
11453
|
} else {
|
|
11229
11454
|
console.warn("Unsupported AI SDK model. Not wrapping.");
|
|
@@ -11279,10 +11504,10 @@ var BraintrustLanguageModelWrapper = class {
|
|
|
11279
11504
|
metrics: {
|
|
11280
11505
|
time_to_first_token: getCurrentUnixTimestamp() - startTime,
|
|
11281
11506
|
tokens: !isEmpty(ret.usage) ? ret.usage.promptTokens + ret.usage.completionTokens : void 0,
|
|
11282
|
-
prompt_tokens: _optionalChain([ret, 'access',
|
|
11283
|
-
completion_tokens: _optionalChain([ret, 'access',
|
|
11507
|
+
prompt_tokens: _optionalChain([ret, 'access', _187 => _187.usage, 'optionalAccess', _188 => _188.promptTokens]),
|
|
11508
|
+
completion_tokens: _optionalChain([ret, 'access', _189 => _189.usage, 'optionalAccess', _190 => _190.completionTokens]),
|
|
11284
11509
|
cached: parseCachedHeader(
|
|
11285
|
-
_nullishCoalesce(_optionalChain([ret, 'access',
|
|
11510
|
+
_nullishCoalesce(_optionalChain([ret, 'access', _191 => _191.rawResponse, 'optionalAccess', _192 => _192.headers, 'optionalAccess', _193 => _193[X_CACHED_HEADER]]), () => ( _optionalChain([ret, 'access', _194 => _194.rawResponse, 'optionalAccess', _195 => _195.headers, 'optionalAccess', _196 => _196[LEGACY_CACHED_HEADER]])))
|
|
11286
11511
|
)
|
|
11287
11512
|
}
|
|
11288
11513
|
});
|
|
@@ -11374,10 +11599,10 @@ var BraintrustLanguageModelWrapper = class {
|
|
|
11374
11599
|
metrics: {
|
|
11375
11600
|
time_to_first_token,
|
|
11376
11601
|
tokens: !isEmpty(usage) ? usage.promptTokens + usage.completionTokens : void 0,
|
|
11377
|
-
prompt_tokens: _optionalChain([usage, 'optionalAccess',
|
|
11378
|
-
completion_tokens: _optionalChain([usage, 'optionalAccess',
|
|
11602
|
+
prompt_tokens: _optionalChain([usage, 'optionalAccess', _197 => _197.promptTokens]),
|
|
11603
|
+
completion_tokens: _optionalChain([usage, 'optionalAccess', _198 => _198.completionTokens]),
|
|
11379
11604
|
cached: parseCachedHeader(
|
|
11380
|
-
_nullishCoalesce(_optionalChain([ret, 'access',
|
|
11605
|
+
_nullishCoalesce(_optionalChain([ret, 'access', _199 => _199.rawResponse, 'optionalAccess', _200 => _200.headers, 'optionalAccess', _201 => _201[X_CACHED_HEADER]]), () => ( _optionalChain([ret, 'access', _202 => _202.rawResponse, 'optionalAccess', _203 => _203.headers, 'optionalAccess', _204 => _204[LEGACY_CACHED_HEADER]])))
|
|
11381
11606
|
)
|
|
11382
11607
|
}
|
|
11383
11608
|
});
|
|
@@ -11424,7 +11649,7 @@ function postProcessPrompt(prompt) {
|
|
|
11424
11649
|
return [
|
|
11425
11650
|
{
|
|
11426
11651
|
role: "assistant",
|
|
11427
|
-
content: _optionalChain([textPart, 'optionalAccess',
|
|
11652
|
+
content: _optionalChain([textPart, 'optionalAccess', _205 => _205.text]),
|
|
11428
11653
|
...toolCallParts.length > 0 ? {
|
|
11429
11654
|
tool_calls: toolCallParts.map((part) => ({
|
|
11430
11655
|
id: part.toolCallId,
|
|
@@ -11649,9 +11874,9 @@ function streamNextProxy(stream, sspan) {
|
|
|
11649
11874
|
return result;
|
|
11650
11875
|
}
|
|
11651
11876
|
const item = result.value;
|
|
11652
|
-
switch (_optionalChain([item, 'optionalAccess',
|
|
11877
|
+
switch (_optionalChain([item, 'optionalAccess', _206 => _206.type])) {
|
|
11653
11878
|
case "message_start":
|
|
11654
|
-
const msg = _optionalChain([item, 'optionalAccess',
|
|
11879
|
+
const msg = _optionalChain([item, 'optionalAccess', _207 => _207.message]);
|
|
11655
11880
|
if (msg) {
|
|
11656
11881
|
const event = parseEventFromMessage(msg);
|
|
11657
11882
|
totals = { ...totals, ...event.metrics };
|
|
@@ -11659,20 +11884,20 @@ function streamNextProxy(stream, sspan) {
|
|
|
11659
11884
|
}
|
|
11660
11885
|
break;
|
|
11661
11886
|
case "content_block_delta":
|
|
11662
|
-
if (_optionalChain([item, 'access',
|
|
11663
|
-
const text = _optionalChain([item, 'optionalAccess',
|
|
11887
|
+
if (_optionalChain([item, 'access', _208 => _208.delta, 'optionalAccess', _209 => _209.type]) === "text_delta") {
|
|
11888
|
+
const text = _optionalChain([item, 'optionalAccess', _210 => _210.delta, 'optionalAccess', _211 => _211.text]);
|
|
11664
11889
|
if (text) {
|
|
11665
11890
|
deltas.push(text);
|
|
11666
11891
|
}
|
|
11667
11892
|
}
|
|
11668
11893
|
break;
|
|
11669
11894
|
case "message_delta":
|
|
11670
|
-
const usage = _optionalChain([item, 'optionalAccess',
|
|
11895
|
+
const usage = _optionalChain([item, 'optionalAccess', _212 => _212.usage]);
|
|
11671
11896
|
if (usage) {
|
|
11672
11897
|
const metrics = parseMetricsFromUsage2(usage);
|
|
11673
11898
|
totals = { ...totals, ...metrics };
|
|
11674
11899
|
}
|
|
11675
|
-
const delta = _optionalChain([item, 'optionalAccess',
|
|
11900
|
+
const delta = _optionalChain([item, 'optionalAccess', _213 => _213.delta]);
|
|
11676
11901
|
if (delta) {
|
|
11677
11902
|
metadata = { ...metadata, ...delta };
|
|
11678
11903
|
}
|
|
@@ -11684,8 +11909,8 @@ function streamNextProxy(stream, sspan) {
|
|
|
11684
11909
|
};
|
|
11685
11910
|
}
|
|
11686
11911
|
function parseEventFromMessage(message) {
|
|
11687
|
-
const output = _optionalChain([message, 'optionalAccess',
|
|
11688
|
-
const metrics = parseMetricsFromUsage2(_optionalChain([message, 'optionalAccess',
|
|
11912
|
+
const output = _optionalChain([message, 'optionalAccess', _214 => _214.content]) || null;
|
|
11913
|
+
const metrics = parseMetricsFromUsage2(_optionalChain([message, 'optionalAccess', _215 => _215.usage]));
|
|
11689
11914
|
const metas = ["stop_reason", "stop_sequence"];
|
|
11690
11915
|
const metadata = {};
|
|
11691
11916
|
for (const m of metas) {
|
|
@@ -11976,6 +12201,8 @@ var evalBodySchema = _zod.z.object({
|
|
|
11976
12201
|
name: _zod.z.string()
|
|
11977
12202
|
})
|
|
11978
12203
|
).nullish(),
|
|
12204
|
+
experiment_name: _zod.z.string().nullish(),
|
|
12205
|
+
project_id: _zod.z.string().nullish(),
|
|
11979
12206
|
parent: _typespecs.invokeParent.optional(),
|
|
11980
12207
|
stream: _zod.z.boolean().optional()
|
|
11981
12208
|
});
|
|
@@ -12103,4 +12330,5 @@ var index_default = exports_node_exports;
|
|
|
12103
12330
|
|
|
12104
12331
|
|
|
12105
12332
|
|
|
12106
|
-
|
|
12333
|
+
|
|
12334
|
+
exports.AISpanProcessor = AISpanProcessor; exports.Attachment = Attachment; exports.BaseAttachment = BaseAttachment; exports.BaseExperiment = BaseExperiment; exports.BraintrustExporter = BraintrustExporter; exports.BraintrustMiddleware = BraintrustMiddleware; exports.BraintrustSpanProcessor = BraintrustSpanProcessor; exports.BraintrustState = BraintrustState; exports.BraintrustStream = BraintrustStream; exports.CodeFunction = CodeFunction; exports.CodePrompt = CodePrompt; exports.Dataset = Dataset; exports.ERR_PERMALINK = ERR_PERMALINK; exports.Eval = Eval; exports.EvalResultWithSummary = EvalResultWithSummary; exports.Experiment = Experiment; exports.ExternalAttachment = ExternalAttachment; exports.FailedHTTPResponse = FailedHTTPResponse; exports.INTERNAL_BTQL_LIMIT = INTERNAL_BTQL_LIMIT; exports.LEGACY_CACHED_HEADER = LEGACY_CACHED_HEADER; exports.LazyValue = LazyValue; exports.Logger = Logger; exports.NOOP_SPAN = NOOP_SPAN; exports.NOOP_SPAN_PERMALINK = NOOP_SPAN_PERMALINK; exports.NoopSpan = NoopSpan; exports.Project = Project; exports.ProjectNameIdMap = ProjectNameIdMap; exports.Prompt = Prompt; exports.PromptBuilder = PromptBuilder; exports.ReadonlyAttachment = ReadonlyAttachment; exports.ReadonlyExperiment = ReadonlyExperiment; exports.Reporter = Reporter; exports.ScorerBuilder = ScorerBuilder; exports.SpanImpl = SpanImpl; exports.TestBackgroundLogger = TestBackgroundLogger; exports.ToolBuilder = ToolBuilder; exports.X_CACHED_HEADER = X_CACHED_HEADER; exports._exportsForTestingOnly = _exportsForTestingOnly; exports._internalGetGlobalState = _internalGetGlobalState; exports._internalSetInitialState = _internalSetInitialState; exports.braintrustStreamChunkSchema = braintrustStreamChunkSchema; exports.buildLocalSummary = buildLocalSummary; exports.createFinalValuePassThroughStream = createFinalValuePassThroughStream; exports.currentExperiment = currentExperiment; exports.currentLogger = currentLogger; exports.currentSpan = currentSpan; exports.default = index_default; exports.defaultErrorScoreHandler = defaultErrorScoreHandler; exports.deserializePlainStringAsJSON = deserializePlainStringAsJSON; exports.devNullWritableStream = devNullWritableStream; exports.evaluatorDefinitionSchema = evaluatorDefinitionSchema; exports.evaluatorDefinitionsSchema = evaluatorDefinitionsSchema; exports.flush = flush; exports.getSpanParentObject = getSpanParentObject; exports.graph = graph_framework_exports; exports.init = init; exports.initDataset = initDataset; exports.initExperiment = initExperiment; exports.initFunction = initFunction; exports.initLogger = initLogger; exports.invoke = invoke; exports.loadPrompt = loadPrompt; exports.log = log; exports.logError = logError; exports.login = login; exports.loginToState = loginToState; exports.newId = newId; exports.parseCachedHeader = parseCachedHeader; exports.permalink = permalink; exports.projects = projects; exports.promptContentsSchema = promptContentsSchema; exports.promptDefinitionSchema = promptDefinitionSchema; exports.promptDefinitionToPromptData = promptDefinitionToPromptData; exports.promptDefinitionWithToolsSchema = promptDefinitionWithToolsSchema; exports.renderMessage = renderMessage; exports.renderPromptParams = renderPromptParams; exports.reportFailures = reportFailures; exports.runEvaluator = runEvaluator; exports.setFetch = setFetch; exports.spanComponentsToObjectId = spanComponentsToObjectId; exports.startSpan = startSpan; exports.summarize = summarize; exports.toolFunctionDefinitionSchema = _typespecs.toolFunctionDefinitionSchema; exports.traceable = traceable; exports.traced = traced; exports.updateSpan = updateSpan; exports.withCurrent = withCurrent; exports.withDataset = withDataset; exports.withExperiment = withExperiment; exports.withLogger = withLogger; exports.withParent = withParent; exports.wrapAISDKModel = wrapAISDKModel; exports.wrapAnthropic = wrapAnthropic; exports.wrapOpenAI = wrapOpenAI; exports.wrapOpenAIv4 = wrapOpenAIv4; exports.wrapTraced = wrapTraced;
|
package/dist/index.mjs
CHANGED
|
@@ -7979,6 +7979,11 @@ var Prompt = class _Prompt {
|
|
|
7979
7979
|
}
|
|
7980
7980
|
};
|
|
7981
7981
|
var TEST_API_KEY = "___TEST_API_KEY__THIS_IS_NOT_REAL___";
|
|
7982
|
+
function setInitialTestState() {
|
|
7983
|
+
if (!_internalGetGlobalState()) {
|
|
7984
|
+
_internalSetInitialState();
|
|
7985
|
+
}
|
|
7986
|
+
}
|
|
7982
7987
|
async function simulateLoginForTests() {
|
|
7983
7988
|
return await login({
|
|
7984
7989
|
apiKey: TEST_API_KEY,
|
|
@@ -7996,7 +8001,8 @@ var _exportsForTestingOnly = {
|
|
|
7996
8001
|
useTestBackgroundLogger,
|
|
7997
8002
|
clearTestBackgroundLogger,
|
|
7998
8003
|
simulateLoginForTests,
|
|
7999
|
-
simulateLogoutForTests
|
|
8004
|
+
simulateLogoutForTests,
|
|
8005
|
+
setInitialTestState
|
|
8000
8006
|
};
|
|
8001
8007
|
|
|
8002
8008
|
// src/node.ts
|
|
@@ -8035,6 +8041,7 @@ __export(exports_node_exports, {
|
|
|
8035
8041
|
BaseAttachment: () => BaseAttachment,
|
|
8036
8042
|
BaseExperiment: () => BaseExperiment,
|
|
8037
8043
|
BraintrustExporter: () => BraintrustExporter,
|
|
8044
|
+
BraintrustMiddleware: () => BraintrustMiddleware,
|
|
8038
8045
|
BraintrustSpanProcessor: () => BraintrustSpanProcessor,
|
|
8039
8046
|
BraintrustState: () => BraintrustState,
|
|
8040
8047
|
BraintrustStream: () => BraintrustStream,
|
|
@@ -10841,10 +10848,17 @@ function wrapOpenAI(openai) {
|
|
|
10841
10848
|
}
|
|
10842
10849
|
globalThis.__inherited_braintrust_wrap_openai = wrapOpenAI;
|
|
10843
10850
|
function wrapOpenAIv4(openai) {
|
|
10844
|
-
const completionProxy =
|
|
10845
|
-
|
|
10846
|
-
|
|
10847
|
-
|
|
10851
|
+
const completionProxy = new Proxy(openai.chat.completions, {
|
|
10852
|
+
get(target, name, receiver) {
|
|
10853
|
+
const baseVal = Reflect.get(target, name, receiver);
|
|
10854
|
+
if (name === "create") {
|
|
10855
|
+
return wrapChatCompletion(baseVal.bind(target));
|
|
10856
|
+
} else if (name === "parse") {
|
|
10857
|
+
return wrapBetaChatCompletionParse(baseVal.bind(target));
|
|
10858
|
+
}
|
|
10859
|
+
return baseVal;
|
|
10860
|
+
}
|
|
10861
|
+
});
|
|
10848
10862
|
const chatProxy = new Proxy(openai.chat, {
|
|
10849
10863
|
get(target, name, receiver) {
|
|
10850
10864
|
if (name === "completions") {
|
|
@@ -11220,7 +11234,218 @@ var WrapperStream = class {
|
|
|
11220
11234
|
}
|
|
11221
11235
|
};
|
|
11222
11236
|
|
|
11223
|
-
// src/wrappers/ai-sdk.ts
|
|
11237
|
+
// src/wrappers/ai-sdk-v2.ts
|
|
11238
|
+
import { SpanTypeAttribute as SpanTypeAttribute4 } from "@braintrust/core";
|
|
11239
|
+
function detectProviderFromResult(result) {
|
|
11240
|
+
if (!result?.providerMetadata) {
|
|
11241
|
+
return void 0;
|
|
11242
|
+
}
|
|
11243
|
+
const keys = Object.keys(result.providerMetadata);
|
|
11244
|
+
return keys?.at(0);
|
|
11245
|
+
}
|
|
11246
|
+
function extractModelFromResult(result) {
|
|
11247
|
+
if (result?.response?.modelId) {
|
|
11248
|
+
return result.response.modelId;
|
|
11249
|
+
}
|
|
11250
|
+
if (result?.request?.body?.model) {
|
|
11251
|
+
return result.request.body.model;
|
|
11252
|
+
}
|
|
11253
|
+
return void 0;
|
|
11254
|
+
}
|
|
11255
|
+
function camelToSnake(str) {
|
|
11256
|
+
return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
|
|
11257
|
+
}
|
|
11258
|
+
function extractModelParameters(params) {
|
|
11259
|
+
const modelParams = {};
|
|
11260
|
+
const excludeKeys = /* @__PURE__ */ new Set([
|
|
11261
|
+
"prompt",
|
|
11262
|
+
// Already captured as input
|
|
11263
|
+
"system",
|
|
11264
|
+
// Already captured as input
|
|
11265
|
+
"messages",
|
|
11266
|
+
// Already captured as input
|
|
11267
|
+
"model",
|
|
11268
|
+
// Already captured in metadata.model
|
|
11269
|
+
"providerOptions"
|
|
11270
|
+
// Internal AI SDK configuration
|
|
11271
|
+
]);
|
|
11272
|
+
for (const [key, value] of Object.entries(params)) {
|
|
11273
|
+
if (value !== void 0 && !excludeKeys.has(key)) {
|
|
11274
|
+
const snakeKey = camelToSnake(key);
|
|
11275
|
+
modelParams[snakeKey] = value;
|
|
11276
|
+
}
|
|
11277
|
+
}
|
|
11278
|
+
return modelParams;
|
|
11279
|
+
}
|
|
11280
|
+
function getNumberProperty(obj, key) {
|
|
11281
|
+
if (!obj || typeof obj !== "object" || !(key in obj)) {
|
|
11282
|
+
return void 0;
|
|
11283
|
+
}
|
|
11284
|
+
const value = Reflect.get(obj, key);
|
|
11285
|
+
return typeof value === "number" ? value : void 0;
|
|
11286
|
+
}
|
|
11287
|
+
function normalizeUsageMetrics(usage) {
|
|
11288
|
+
const metrics = {};
|
|
11289
|
+
const inputTokens = getNumberProperty(usage, "inputTokens");
|
|
11290
|
+
if (inputTokens !== void 0) {
|
|
11291
|
+
metrics.prompt_tokens = inputTokens;
|
|
11292
|
+
}
|
|
11293
|
+
const outputTokens = getNumberProperty(usage, "outputTokens");
|
|
11294
|
+
if (outputTokens !== void 0) {
|
|
11295
|
+
metrics.completion_tokens = outputTokens;
|
|
11296
|
+
}
|
|
11297
|
+
const totalTokens = getNumberProperty(usage, "totalTokens");
|
|
11298
|
+
if (totalTokens !== void 0) {
|
|
11299
|
+
metrics.tokens = totalTokens;
|
|
11300
|
+
}
|
|
11301
|
+
const reasoningTokens = getNumberProperty(usage, "reasoningTokens");
|
|
11302
|
+
if (reasoningTokens !== void 0) {
|
|
11303
|
+
metrics.completion_reasoning_tokens = reasoningTokens;
|
|
11304
|
+
}
|
|
11305
|
+
const cachedInputTokens = getNumberProperty(usage, "cachedInputTokens");
|
|
11306
|
+
if (cachedInputTokens !== void 0) {
|
|
11307
|
+
metrics.prompt_cached_tokens = cachedInputTokens;
|
|
11308
|
+
}
|
|
11309
|
+
return metrics;
|
|
11310
|
+
}
|
|
11311
|
+
function BraintrustMiddleware(config = {}) {
|
|
11312
|
+
return {
|
|
11313
|
+
wrapGenerate: async ({ doGenerate, params }) => {
|
|
11314
|
+
const spanArgs = {
|
|
11315
|
+
name: "ai-sdk.generateText",
|
|
11316
|
+
spanAttributes: {
|
|
11317
|
+
type: SpanTypeAttribute4.LLM
|
|
11318
|
+
},
|
|
11319
|
+
event: {
|
|
11320
|
+
input: params.prompt,
|
|
11321
|
+
metadata: {
|
|
11322
|
+
...extractModelParameters(params)
|
|
11323
|
+
}
|
|
11324
|
+
}
|
|
11325
|
+
};
|
|
11326
|
+
const span = startSpan(spanArgs);
|
|
11327
|
+
try {
|
|
11328
|
+
const result = await doGenerate();
|
|
11329
|
+
const metadata = {};
|
|
11330
|
+
const provider = detectProviderFromResult(result);
|
|
11331
|
+
if (provider !== void 0) {
|
|
11332
|
+
metadata.provider = provider;
|
|
11333
|
+
}
|
|
11334
|
+
if (result.finishReason !== void 0) {
|
|
11335
|
+
metadata.finish_reason = result.finishReason;
|
|
11336
|
+
}
|
|
11337
|
+
const model = extractModelFromResult(result);
|
|
11338
|
+
if (model !== void 0) {
|
|
11339
|
+
metadata.model = model;
|
|
11340
|
+
}
|
|
11341
|
+
span.log({
|
|
11342
|
+
output: result.content,
|
|
11343
|
+
metadata,
|
|
11344
|
+
metrics: normalizeUsageMetrics(result.usage)
|
|
11345
|
+
});
|
|
11346
|
+
return result;
|
|
11347
|
+
} catch (error2) {
|
|
11348
|
+
span.log({
|
|
11349
|
+
error: error2 instanceof Error ? error2.message : String(error2)
|
|
11350
|
+
});
|
|
11351
|
+
throw error2;
|
|
11352
|
+
} finally {
|
|
11353
|
+
span.end();
|
|
11354
|
+
}
|
|
11355
|
+
},
|
|
11356
|
+
wrapStream: async ({ doStream, params }) => {
|
|
11357
|
+
const spanArgs = {
|
|
11358
|
+
name: "ai-sdk.streamText",
|
|
11359
|
+
spanAttributes: {
|
|
11360
|
+
type: SpanTypeAttribute4.LLM
|
|
11361
|
+
},
|
|
11362
|
+
event: {
|
|
11363
|
+
input: params.prompt,
|
|
11364
|
+
metadata: {
|
|
11365
|
+
...extractModelParameters(params)
|
|
11366
|
+
}
|
|
11367
|
+
}
|
|
11368
|
+
};
|
|
11369
|
+
const span = startSpan(spanArgs);
|
|
11370
|
+
try {
|
|
11371
|
+
const { stream, ...rest } = await doStream();
|
|
11372
|
+
const textChunks = [];
|
|
11373
|
+
let finalUsage = {};
|
|
11374
|
+
let finalFinishReason = void 0;
|
|
11375
|
+
let providerMetadata = {};
|
|
11376
|
+
const transformStream = new TransformStream({
|
|
11377
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
11378
|
+
transform(chunk, controller) {
|
|
11379
|
+
try {
|
|
11380
|
+
if (chunk.type === "text-delta" && chunk.delta) {
|
|
11381
|
+
textChunks.push(chunk.delta);
|
|
11382
|
+
}
|
|
11383
|
+
if (chunk.type === "finish") {
|
|
11384
|
+
finalFinishReason = chunk.finishReason;
|
|
11385
|
+
finalUsage = chunk.usage || {};
|
|
11386
|
+
providerMetadata = chunk.providerMetadata || {};
|
|
11387
|
+
}
|
|
11388
|
+
controller.enqueue(chunk);
|
|
11389
|
+
} catch (error2) {
|
|
11390
|
+
span.log({
|
|
11391
|
+
error: error2 instanceof Error ? error2.message : String(error2)
|
|
11392
|
+
});
|
|
11393
|
+
span.end();
|
|
11394
|
+
controller.error(error2);
|
|
11395
|
+
}
|
|
11396
|
+
},
|
|
11397
|
+
flush() {
|
|
11398
|
+
try {
|
|
11399
|
+
const generatedText = textChunks.join("");
|
|
11400
|
+
const output = generatedText ? [{ type: "text", text: generatedText }] : [];
|
|
11401
|
+
const resultForDetection = {
|
|
11402
|
+
providerMetadata,
|
|
11403
|
+
response: rest.response,
|
|
11404
|
+
...rest
|
|
11405
|
+
};
|
|
11406
|
+
const metadata = {};
|
|
11407
|
+
const provider = detectProviderFromResult(resultForDetection);
|
|
11408
|
+
if (provider !== void 0) {
|
|
11409
|
+
metadata.provider = provider;
|
|
11410
|
+
}
|
|
11411
|
+
if (finalFinishReason !== void 0) {
|
|
11412
|
+
metadata.finish_reason = finalFinishReason;
|
|
11413
|
+
}
|
|
11414
|
+
const model = extractModelFromResult(resultForDetection);
|
|
11415
|
+
if (model !== void 0) {
|
|
11416
|
+
metadata.model = model;
|
|
11417
|
+
}
|
|
11418
|
+
span.log({
|
|
11419
|
+
output,
|
|
11420
|
+
metadata,
|
|
11421
|
+
metrics: normalizeUsageMetrics(finalUsage)
|
|
11422
|
+
});
|
|
11423
|
+
span.end();
|
|
11424
|
+
} catch (error2) {
|
|
11425
|
+
span.log({
|
|
11426
|
+
error: error2 instanceof Error ? error2.message : String(error2)
|
|
11427
|
+
});
|
|
11428
|
+
span.end();
|
|
11429
|
+
throw error2;
|
|
11430
|
+
}
|
|
11431
|
+
}
|
|
11432
|
+
});
|
|
11433
|
+
return {
|
|
11434
|
+
stream: stream.pipeThrough(transformStream),
|
|
11435
|
+
...rest
|
|
11436
|
+
};
|
|
11437
|
+
} catch (error2) {
|
|
11438
|
+
span.log({
|
|
11439
|
+
error: error2 instanceof Error ? error2.message : String(error2)
|
|
11440
|
+
});
|
|
11441
|
+
span.end();
|
|
11442
|
+
throw error2;
|
|
11443
|
+
}
|
|
11444
|
+
}
|
|
11445
|
+
};
|
|
11446
|
+
}
|
|
11447
|
+
|
|
11448
|
+
// src/wrappers/ai-sdk-v1.ts
|
|
11224
11449
|
function wrapAISDKModel(model) {
|
|
11225
11450
|
const m = model;
|
|
11226
11451
|
if (m?.specificationVersion === "v1" && typeof m?.provider === "string" && typeof m?.modelId === "string") {
|
|
@@ -11484,7 +11709,7 @@ function postProcessOutput(text, toolCalls, finishReason) {
|
|
|
11484
11709
|
}
|
|
11485
11710
|
|
|
11486
11711
|
// src/wrappers/anthropic.ts
|
|
11487
|
-
import { SpanTypeAttribute as
|
|
11712
|
+
import { SpanTypeAttribute as SpanTypeAttribute5 } from "@braintrust/core";
|
|
11488
11713
|
function wrapAnthropic(anthropic) {
|
|
11489
11714
|
const au = anthropic;
|
|
11490
11715
|
if (au && typeof au === "object" && "messages" in au && typeof au.messages === "object" && au.messages && "create" in au.messages) {
|
|
@@ -11542,7 +11767,7 @@ function createProxy(create) {
|
|
|
11542
11767
|
const spanArgs = {
|
|
11543
11768
|
name: "anthropic.messages.create",
|
|
11544
11769
|
spanAttributes: {
|
|
11545
|
-
type:
|
|
11770
|
+
type: SpanTypeAttribute5.LLM
|
|
11546
11771
|
},
|
|
11547
11772
|
event: {
|
|
11548
11773
|
input,
|
|
@@ -11976,6 +12201,8 @@ var evalBodySchema = z5.object({
|
|
|
11976
12201
|
name: z5.string()
|
|
11977
12202
|
})
|
|
11978
12203
|
).nullish(),
|
|
12204
|
+
experiment_name: z5.string().nullish(),
|
|
12205
|
+
project_id: z5.string().nullish(),
|
|
11979
12206
|
parent: invokeParent.optional(),
|
|
11980
12207
|
stream: z5.boolean().optional()
|
|
11981
12208
|
});
|
|
@@ -12013,6 +12240,7 @@ export {
|
|
|
12013
12240
|
BaseAttachment,
|
|
12014
12241
|
BaseExperiment,
|
|
12015
12242
|
BraintrustExporter,
|
|
12243
|
+
BraintrustMiddleware,
|
|
12016
12244
|
BraintrustSpanProcessor,
|
|
12017
12245
|
BraintrustState,
|
|
12018
12246
|
BraintrustStream,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "braintrust",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "SDK for integrating Braintrust",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -51,11 +51,13 @@
|
|
|
51
51
|
"docs": "npx typedoc --options typedoc.json src/index.ts",
|
|
52
52
|
"prepublishOnly": "../../scripts/node_prepublish_sdk.py",
|
|
53
53
|
"postpublish": "../../scripts/node_postpublish_sdk.py",
|
|
54
|
-
"test": "vitest run --exclude src/wrappers/anthropic.test.ts --exclude src/wrappers/oai.test.ts --exclude src/otel.test.ts --exclude src/otel-no-deps.test.ts",
|
|
54
|
+
"test": "vitest run --exclude src/wrappers/anthropic.test.ts --exclude src/wrappers/oai.test.ts --exclude src/otel.test.ts --exclude src/otel-no-deps.test.ts --exclude src/wrappers/ai-sdk-v1.test.ts --exclude src/wrappers/ai-sdk-v2.test.ts",
|
|
55
55
|
"test:anthropic": "vitest run src/wrappers/anthropic.test.ts",
|
|
56
56
|
"test:openai": "vitest run src/wrappers/oai.test.ts",
|
|
57
57
|
"test:otel": "vitest run src/otel.test.ts",
|
|
58
|
-
"test:otel-no-deps": "vitest run src/otel-no-deps.test.ts --reporter=verbose"
|
|
58
|
+
"test:otel-no-deps": "vitest run src/otel-no-deps.test.ts --reporter=verbose",
|
|
59
|
+
"test:ai-sdk-v1": "vitest run src/wrappers/ai-sdk-v1.test.ts",
|
|
60
|
+
"test:ai-sdk-v2": "vitest run src/wrappers/ai-sdk-v2.test.ts src/wrappers/ai-sdk-v1.test.ts"
|
|
59
61
|
},
|
|
60
62
|
"author": "",
|
|
61
63
|
"license": "MIT",
|