braintrust 3.27.0 → 3.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dev/dist/index.d.mts +519 -186
- package/dev/dist/index.d.ts +519 -186
- package/dev/dist/index.js +1837 -1009
- package/dev/dist/index.mjs +1172 -344
- package/dist/apply-auto-instrumentation.js +262 -210
- package/dist/apply-auto-instrumentation.mjs +54 -2
- package/dist/auto-instrumentations/bundler/esbuild.cjs +81 -2
- package/dist/auto-instrumentations/bundler/esbuild.mjs +2 -2
- package/dist/auto-instrumentations/bundler/next.cjs +81 -2
- package/dist/auto-instrumentations/bundler/next.mjs +3 -3
- package/dist/auto-instrumentations/bundler/rollup.cjs +81 -2
- package/dist/auto-instrumentations/bundler/rollup.mjs +2 -2
- package/dist/auto-instrumentations/bundler/vite.cjs +81 -2
- package/dist/auto-instrumentations/bundler/vite.mjs +2 -2
- package/dist/auto-instrumentations/bundler/webpack-loader.cjs +81 -2
- package/dist/auto-instrumentations/bundler/webpack.cjs +81 -2
- package/dist/auto-instrumentations/bundler/webpack.mjs +3 -3
- package/dist/auto-instrumentations/{chunk-XEYKUBLY.mjs → chunk-26PKVUKB.mjs} +80 -2
- package/dist/auto-instrumentations/{chunk-BW33ULMW.mjs → chunk-HD35AM3M.mjs} +1 -1
- package/dist/auto-instrumentations/{chunk-ZNHTSSGI.mjs → chunk-NP7V4XB2.mjs} +2 -1
- package/dist/auto-instrumentations/hook.mjs +236 -30
- package/dist/auto-instrumentations/index.cjs +2 -1
- package/dist/auto-instrumentations/index.mjs +1 -1
- package/dist/browser.d.mts +628 -53
- package/dist/browser.d.ts +628 -53
- package/dist/browser.js +2301 -284
- package/dist/browser.mjs +2301 -284
- package/dist/{chunk-MF7NU6BT.js → chunk-BBE7SNRV.js} +34 -4
- package/dist/{chunk-QRHGVBKU.js → chunk-OBBWQW6K.js} +1799 -1023
- package/dist/{chunk-YKD22IMR.mjs → chunk-UPFNQCGB.mjs} +966 -190
- package/dist/{chunk-CZM5JIQL.mjs → chunk-ZHUHZWFY.mjs} +33 -3
- package/dist/cli.js +1224 -398
- package/dist/edge-light.d.mts +1 -1
- package/dist/edge-light.d.ts +1 -1
- package/dist/edge-light.js +2301 -284
- package/dist/edge-light.mjs +2301 -284
- package/dist/index.d.mts +1212 -637
- package/dist/index.d.ts +1212 -637
- package/dist/index.js +1880 -590
- package/dist/index.mjs +1450 -160
- package/dist/instrumentation/index.d.mts +190 -6
- package/dist/instrumentation/index.d.ts +190 -6
- package/dist/instrumentation/index.js +823 -116
- package/dist/instrumentation/index.mjs +823 -116
- package/dist/vitest-evals-reporter.js +16 -16
- package/dist/vitest-evals-reporter.mjs +2 -2
- package/dist/workerd.d.mts +1 -1
- package/dist/workerd.d.ts +1 -1
- package/dist/workerd.js +2301 -284
- package/dist/workerd.mjs +2301 -284
- package/package.json +2 -3
- package/util/dist/index.d.mts +1545 -100
- package/util/dist/index.d.ts +1545 -100
package/dist/cli.js
CHANGED
|
@@ -41,8 +41,8 @@ var require_async = __commonJS({
|
|
|
41
41
|
"use strict";
|
|
42
42
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
43
43
|
exports2.read = void 0;
|
|
44
|
-
function read(
|
|
45
|
-
settings.fs.lstat(
|
|
44
|
+
function read(path9, settings, callback) {
|
|
45
|
+
settings.fs.lstat(path9, (lstatError, lstat) => {
|
|
46
46
|
if (lstatError !== null) {
|
|
47
47
|
callFailureCallback(callback, lstatError);
|
|
48
48
|
return;
|
|
@@ -51,7 +51,7 @@ var require_async = __commonJS({
|
|
|
51
51
|
callSuccessCallback(callback, lstat);
|
|
52
52
|
return;
|
|
53
53
|
}
|
|
54
|
-
settings.fs.stat(
|
|
54
|
+
settings.fs.stat(path9, (statError, stat2) => {
|
|
55
55
|
if (statError !== null) {
|
|
56
56
|
if (settings.throwErrorOnBrokenSymbolicLink) {
|
|
57
57
|
callFailureCallback(callback, statError);
|
|
@@ -83,13 +83,13 @@ var require_sync = __commonJS({
|
|
|
83
83
|
"use strict";
|
|
84
84
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
85
85
|
exports2.read = void 0;
|
|
86
|
-
function read(
|
|
87
|
-
const lstat = settings.fs.lstatSync(
|
|
86
|
+
function read(path9, settings) {
|
|
87
|
+
const lstat = settings.fs.lstatSync(path9);
|
|
88
88
|
if (!lstat.isSymbolicLink() || !settings.followSymbolicLink) {
|
|
89
89
|
return lstat;
|
|
90
90
|
}
|
|
91
91
|
try {
|
|
92
|
-
const stat2 = settings.fs.statSync(
|
|
92
|
+
const stat2 = settings.fs.statSync(path9);
|
|
93
93
|
if (settings.markSymbolicLink) {
|
|
94
94
|
stat2.isSymbolicLink = () => true;
|
|
95
95
|
}
|
|
@@ -160,17 +160,17 @@ var require_out = __commonJS({
|
|
|
160
160
|
var sync = require_sync();
|
|
161
161
|
var settings_1 = require_settings();
|
|
162
162
|
exports2.Settings = settings_1.default;
|
|
163
|
-
function stat2(
|
|
163
|
+
function stat2(path9, optionsOrSettingsOrCallback, callback) {
|
|
164
164
|
if (typeof optionsOrSettingsOrCallback === "function") {
|
|
165
|
-
async.read(
|
|
165
|
+
async.read(path9, getSettings(), optionsOrSettingsOrCallback);
|
|
166
166
|
return;
|
|
167
167
|
}
|
|
168
|
-
async.read(
|
|
168
|
+
async.read(path9, getSettings(optionsOrSettingsOrCallback), callback);
|
|
169
169
|
}
|
|
170
170
|
exports2.stat = stat2;
|
|
171
|
-
function statSync2(
|
|
171
|
+
function statSync2(path9, optionsOrSettings) {
|
|
172
172
|
const settings = getSettings(optionsOrSettings);
|
|
173
|
-
return sync.read(
|
|
173
|
+
return sync.read(path9, settings);
|
|
174
174
|
}
|
|
175
175
|
exports2.statSync = statSync2;
|
|
176
176
|
function getSettings(settingsOrOptions = {}) {
|
|
@@ -388,16 +388,16 @@ var require_async2 = __commonJS({
|
|
|
388
388
|
return;
|
|
389
389
|
}
|
|
390
390
|
const tasks = names.map((name) => {
|
|
391
|
-
const
|
|
391
|
+
const path9 = common.joinPathSegments(directory, name, settings.pathSegmentSeparator);
|
|
392
392
|
return (done) => {
|
|
393
|
-
fsStat.stat(
|
|
393
|
+
fsStat.stat(path9, settings.fsStatSettings, (error2, stats) => {
|
|
394
394
|
if (error2 !== null) {
|
|
395
395
|
done(error2);
|
|
396
396
|
return;
|
|
397
397
|
}
|
|
398
398
|
const entry = {
|
|
399
399
|
name,
|
|
400
|
-
path:
|
|
400
|
+
path: path9,
|
|
401
401
|
dirent: utils.fs.createDirentFromStats(name, stats)
|
|
402
402
|
};
|
|
403
403
|
if (settings.stats) {
|
|
@@ -515,7 +515,7 @@ var require_settings2 = __commonJS({
|
|
|
515
515
|
"../node_modules/.pnpm/@nodelib+fs.scandir@2.1.5/node_modules/@nodelib/fs.scandir/out/settings.js"(exports2) {
|
|
516
516
|
"use strict";
|
|
517
517
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
518
|
-
var
|
|
518
|
+
var path9 = require("path");
|
|
519
519
|
var fsStat = require_out();
|
|
520
520
|
var fs6 = require_fs3();
|
|
521
521
|
var Settings = class {
|
|
@@ -523,7 +523,7 @@ var require_settings2 = __commonJS({
|
|
|
523
523
|
this._options = _options;
|
|
524
524
|
this.followSymbolicLinks = this._getValue(this._options.followSymbolicLinks, false);
|
|
525
525
|
this.fs = fs6.createFileSystemAdapter(this._options.fs);
|
|
526
|
-
this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator,
|
|
526
|
+
this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator, path9.sep);
|
|
527
527
|
this.stats = this._getValue(this._options.stats, false);
|
|
528
528
|
this.throwErrorOnBrokenSymbolicLink = this._getValue(this._options.throwErrorOnBrokenSymbolicLink, true);
|
|
529
529
|
this.fsStatSettings = new fsStat.Settings({
|
|
@@ -550,17 +550,17 @@ var require_out2 = __commonJS({
|
|
|
550
550
|
var sync = require_sync2();
|
|
551
551
|
var settings_1 = require_settings2();
|
|
552
552
|
exports2.Settings = settings_1.default;
|
|
553
|
-
function scandir(
|
|
553
|
+
function scandir(path9, optionsOrSettingsOrCallback, callback) {
|
|
554
554
|
if (typeof optionsOrSettingsOrCallback === "function") {
|
|
555
|
-
async.read(
|
|
555
|
+
async.read(path9, getSettings(), optionsOrSettingsOrCallback);
|
|
556
556
|
return;
|
|
557
557
|
}
|
|
558
|
-
async.read(
|
|
558
|
+
async.read(path9, getSettings(optionsOrSettingsOrCallback), callback);
|
|
559
559
|
}
|
|
560
560
|
exports2.scandir = scandir;
|
|
561
|
-
function scandirSync(
|
|
561
|
+
function scandirSync(path9, optionsOrSettings) {
|
|
562
562
|
const settings = getSettings(optionsOrSettings);
|
|
563
|
-
return sync.read(
|
|
563
|
+
return sync.read(path9, settings);
|
|
564
564
|
}
|
|
565
565
|
exports2.scandirSync = scandirSync;
|
|
566
566
|
function getSettings(settingsOrOptions = {}) {
|
|
@@ -1164,7 +1164,7 @@ var require_settings3 = __commonJS({
|
|
|
1164
1164
|
"../node_modules/.pnpm/@nodelib+fs.walk@1.2.8/node_modules/@nodelib/fs.walk/out/settings.js"(exports2) {
|
|
1165
1165
|
"use strict";
|
|
1166
1166
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
1167
|
-
var
|
|
1167
|
+
var path9 = require("path");
|
|
1168
1168
|
var fsScandir = require_out2();
|
|
1169
1169
|
var Settings = class {
|
|
1170
1170
|
constructor(_options = {}) {
|
|
@@ -1174,7 +1174,7 @@ var require_settings3 = __commonJS({
|
|
|
1174
1174
|
this.deepFilter = this._getValue(this._options.deepFilter, null);
|
|
1175
1175
|
this.entryFilter = this._getValue(this._options.entryFilter, null);
|
|
1176
1176
|
this.errorFilter = this._getValue(this._options.errorFilter, null);
|
|
1177
|
-
this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator,
|
|
1177
|
+
this.pathSegmentSeparator = this._getValue(this._options.pathSegmentSeparator, path9.sep);
|
|
1178
1178
|
this.fsScandirSettings = new fsScandir.Settings({
|
|
1179
1179
|
followSymbolicLinks: this._options.followSymbolicLinks,
|
|
1180
1180
|
fs: this._options.fs,
|
|
@@ -1236,7 +1236,7 @@ var require_package = __commonJS({
|
|
|
1236
1236
|
"package.json"(exports2, module2) {
|
|
1237
1237
|
module2.exports = {
|
|
1238
1238
|
name: "braintrust",
|
|
1239
|
-
version: "3.
|
|
1239
|
+
version: "3.28.0",
|
|
1240
1240
|
description: "SDK for integrating Braintrust",
|
|
1241
1241
|
repository: {
|
|
1242
1242
|
type: "git",
|
|
@@ -1423,12 +1423,12 @@ var require_package = __commonJS({
|
|
|
1423
1423
|
"@types/mustache": "^4.2.5",
|
|
1424
1424
|
"@types/node": "^20.10.5",
|
|
1425
1425
|
"@types/pluralize": "^0.0.30",
|
|
1426
|
-
"@types/tar": "^6.1.13",
|
|
1427
1426
|
"@typescript-eslint/eslint-plugin": "^8.49.0",
|
|
1428
1427
|
"@typescript-eslint/parser": "^8.49.0",
|
|
1429
1428
|
ai: "^6.0.0",
|
|
1430
1429
|
async: "^3.2.5",
|
|
1431
1430
|
"cross-env": "^7.0.3",
|
|
1431
|
+
eslint: "^9.39.2",
|
|
1432
1432
|
"eslint-plugin-es-x": "9.7.0",
|
|
1433
1433
|
"eslint-plugin-n": "^18.2.1",
|
|
1434
1434
|
"eslint-plugin-node-import": "^1.0.5",
|
|
@@ -1472,7 +1472,6 @@ var require_package = __commonJS({
|
|
|
1472
1472
|
mustache: "^4.2.0",
|
|
1473
1473
|
pluralize: "^8.0.0",
|
|
1474
1474
|
semifies: "^1.0.0",
|
|
1475
|
-
"simple-git": "^3.36.0",
|
|
1476
1475
|
"source-map": "^0.7.4",
|
|
1477
1476
|
"termi-link": "^1.0.1",
|
|
1478
1477
|
unplugin: "^2.3.5",
|
|
@@ -1510,7 +1509,7 @@ __export(cli_exports, {
|
|
|
1510
1509
|
module.exports = __toCommonJS(cli_exports);
|
|
1511
1510
|
var esbuild = __toESM(require("esbuild"));
|
|
1512
1511
|
var dotenv3 = __toESM(require("dotenv"));
|
|
1513
|
-
var
|
|
1512
|
+
var import_node_fs4 = __toESM(require("node:fs"));
|
|
1514
1513
|
var import_node_os = __toESM(require("node:os"));
|
|
1515
1514
|
var import_node_path5 = __toESM(require("node:path"));
|
|
1516
1515
|
var import_node_util5 = __toESM(require("node:util"));
|
|
@@ -3336,15 +3335,15 @@ function mergeDictsWithPaths({
|
|
|
3336
3335
|
function mergeDictsWithPathsHelper({
|
|
3337
3336
|
mergeInto,
|
|
3338
3337
|
mergeFrom,
|
|
3339
|
-
path:
|
|
3338
|
+
path: path9,
|
|
3340
3339
|
mergePaths
|
|
3341
3340
|
}) {
|
|
3342
3341
|
Object.entries(mergeFrom).forEach(([k, mergeFromV]) => {
|
|
3343
3342
|
if (FORBIDDEN_MERGE_KEYS.has(k)) return;
|
|
3344
|
-
const fullPath =
|
|
3343
|
+
const fullPath = path9.concat([k]);
|
|
3345
3344
|
const fullPathSerialized = JSON.stringify(fullPath);
|
|
3346
3345
|
const mergeIntoV = recordFind(mergeInto, k);
|
|
3347
|
-
const isSetUnionField =
|
|
3346
|
+
const isSetUnionField = path9.length === 0 && SET_UNION_FIELDS.has(k) && !mergePaths.has(fullPathSerialized);
|
|
3348
3347
|
if (isSetUnionField && isArray(mergeIntoV) && isArray(mergeFromV)) {
|
|
3349
3348
|
const seen = /* @__PURE__ */ new Set();
|
|
3350
3349
|
const combined = [];
|
|
@@ -3377,9 +3376,9 @@ function mergeDicts(mergeInto, mergeFrom) {
|
|
|
3377
3376
|
function recordFind(m, k) {
|
|
3378
3377
|
return m[k];
|
|
3379
3378
|
}
|
|
3380
|
-
function getObjValueByPath(row,
|
|
3379
|
+
function getObjValueByPath(row, path9) {
|
|
3381
3380
|
let curr = row;
|
|
3382
|
-
for (const p of
|
|
3381
|
+
for (const p of path9) {
|
|
3383
3382
|
if (!isObjectOrArray(curr)) {
|
|
3384
3383
|
return null;
|
|
3385
3384
|
}
|
|
@@ -4015,7 +4014,8 @@ var AclObjectType = import_v36.z.union([
|
|
|
4015
4014
|
"org_member",
|
|
4016
4015
|
"project_log",
|
|
4017
4016
|
"org_project",
|
|
4018
|
-
"org_audit_logs"
|
|
4017
|
+
"org_audit_logs",
|
|
4018
|
+
"project_group"
|
|
4019
4019
|
]),
|
|
4020
4020
|
import_v36.z.null()
|
|
4021
4021
|
]);
|
|
@@ -4144,7 +4144,8 @@ var AsyncScoringState = import_v36.z.union([
|
|
|
4144
4144
|
token: import_v36.z.string(),
|
|
4145
4145
|
function_ids: import_v36.z.array(import_v36.z.unknown()),
|
|
4146
4146
|
skip_logging: import_v36.z.union([import_v36.z.boolean(), import_v36.z.null()]).optional(),
|
|
4147
|
-
triggered_functions: import_v36.z.union([import_v36.z.record(TriggeredFunctionState), import_v36.z.null()]).optional()
|
|
4147
|
+
triggered_functions: import_v36.z.union([import_v36.z.record(TriggeredFunctionState), import_v36.z.null()]).optional(),
|
|
4148
|
+
last_triggered_xact_id: import_v36.z.union([import_v36.z.string(), import_v36.z.number(), import_v36.z.null()]).optional()
|
|
4148
4149
|
}),
|
|
4149
4150
|
import_v36.z.object({ status: import_v36.z.literal("disabled") }),
|
|
4150
4151
|
import_v36.z.null(),
|
|
@@ -4213,7 +4214,7 @@ var FunctionTypeEnum = import_v36.z.enum([
|
|
|
4213
4214
|
"parameters",
|
|
4214
4215
|
"sandbox"
|
|
4215
4216
|
]);
|
|
4216
|
-
var
|
|
4217
|
+
var FacetPreprocessorId = import_v36.z.union([
|
|
4217
4218
|
import_v36.z.object({
|
|
4218
4219
|
type: import_v36.z.literal("function"),
|
|
4219
4220
|
id: import_v36.z.string(),
|
|
@@ -4224,10 +4225,23 @@ var NullableSavedFunctionId = import_v36.z.union([
|
|
|
4224
4225
|
name: import_v36.z.string(),
|
|
4225
4226
|
function_type: FunctionTypeEnum.optional().default("scorer")
|
|
4226
4227
|
}),
|
|
4228
|
+
import_v36.z.object({ type: import_v36.z.literal("inline"), code: import_v36.z.string().min(1) }),
|
|
4227
4229
|
import_v36.z.null()
|
|
4228
4230
|
]);
|
|
4231
|
+
var SavedFunctionId = import_v36.z.union([
|
|
4232
|
+
import_v36.z.object({
|
|
4233
|
+
type: import_v36.z.literal("function"),
|
|
4234
|
+
id: import_v36.z.string(),
|
|
4235
|
+
version: import_v36.z.string().optional()
|
|
4236
|
+
}),
|
|
4237
|
+
import_v36.z.object({
|
|
4238
|
+
type: import_v36.z.literal("global"),
|
|
4239
|
+
name: import_v36.z.string(),
|
|
4240
|
+
function_type: FunctionTypeEnum.optional().default("scorer")
|
|
4241
|
+
})
|
|
4242
|
+
]);
|
|
4229
4243
|
var TopicMapGenerationSettings = import_v36.z.object({
|
|
4230
|
-
algorithm: import_v36.z.enum(["hdbscan", "kmeans"]),
|
|
4244
|
+
algorithm: import_v36.z.enum(["hdbscan", "kmeans", "community"]),
|
|
4231
4245
|
dimension_reduction: import_v36.z.enum(["umap", "pca", "none"]),
|
|
4232
4246
|
sample_size: import_v36.z.number().int().gt(0).optional(),
|
|
4233
4247
|
n_clusters: import_v36.z.number().int().gt(0).optional(),
|
|
@@ -4239,6 +4253,7 @@ var TopicMapGenerationSettings = import_v36.z.object({
|
|
|
4239
4253
|
var TopicMapData = import_v36.z.object({
|
|
4240
4254
|
type: import_v36.z.literal("topic_map"),
|
|
4241
4255
|
source_facet: import_v36.z.string(),
|
|
4256
|
+
source_facet_function: SavedFunctionId.and(import_v36.z.unknown()).optional(),
|
|
4242
4257
|
embedding_model: import_v36.z.string(),
|
|
4243
4258
|
bundle_key: import_v36.z.string().optional(),
|
|
4244
4259
|
report_key: import_v36.z.string().optional(),
|
|
@@ -4252,7 +4267,7 @@ var TopicMapData = import_v36.z.object({
|
|
|
4252
4267
|
});
|
|
4253
4268
|
var BatchedFacetData = import_v36.z.object({
|
|
4254
4269
|
type: import_v36.z.literal("batched_facet"),
|
|
4255
|
-
preprocessor:
|
|
4270
|
+
preprocessor: FacetPreprocessorId.optional(),
|
|
4256
4271
|
facets: import_v36.z.array(
|
|
4257
4272
|
import_v36.z.object({
|
|
4258
4273
|
name: import_v36.z.string(),
|
|
@@ -4513,18 +4528,6 @@ var ObjectReferenceNullish = import_v36.z.union([
|
|
|
4513
4528
|
}),
|
|
4514
4529
|
import_v36.z.null()
|
|
4515
4530
|
]);
|
|
4516
|
-
var SavedFunctionId = import_v36.z.union([
|
|
4517
|
-
import_v36.z.object({
|
|
4518
|
-
type: import_v36.z.literal("function"),
|
|
4519
|
-
id: import_v36.z.string(),
|
|
4520
|
-
version: import_v36.z.string().optional()
|
|
4521
|
-
}),
|
|
4522
|
-
import_v36.z.object({
|
|
4523
|
-
type: import_v36.z.literal("global"),
|
|
4524
|
-
name: import_v36.z.string(),
|
|
4525
|
-
function_type: FunctionTypeEnum.optional().default("scorer")
|
|
4526
|
-
})
|
|
4527
|
-
]);
|
|
4528
4531
|
var DatasetEvent = import_v36.z.object({
|
|
4529
4532
|
id: import_v36.z.string(),
|
|
4530
4533
|
_xact_id: import_v36.z.string(),
|
|
@@ -4746,7 +4749,7 @@ var ExtendedSavedFunctionId = import_v36.z.union([
|
|
|
4746
4749
|
]);
|
|
4747
4750
|
var FacetData = import_v36.z.object({
|
|
4748
4751
|
type: import_v36.z.literal("facet"),
|
|
4749
|
-
preprocessor:
|
|
4752
|
+
preprocessor: FacetPreprocessorId.optional(),
|
|
4750
4753
|
prompt: import_v36.z.string(),
|
|
4751
4754
|
model: import_v36.z.string().optional(),
|
|
4752
4755
|
embedding_model: import_v36.z.string().optional(),
|
|
@@ -5251,6 +5254,19 @@ var MessageRole = import_v36.z.enum([
|
|
|
5251
5254
|
"model",
|
|
5252
5255
|
"developer"
|
|
5253
5256
|
]);
|
|
5257
|
+
var NullableSavedFunctionId = import_v36.z.union([
|
|
5258
|
+
import_v36.z.object({
|
|
5259
|
+
type: import_v36.z.literal("function"),
|
|
5260
|
+
id: import_v36.z.string(),
|
|
5261
|
+
version: import_v36.z.string().optional()
|
|
5262
|
+
}),
|
|
5263
|
+
import_v36.z.object({
|
|
5264
|
+
type: import_v36.z.literal("global"),
|
|
5265
|
+
name: import_v36.z.string(),
|
|
5266
|
+
function_type: FunctionTypeEnum.optional().default("scorer")
|
|
5267
|
+
}),
|
|
5268
|
+
import_v36.z.null()
|
|
5269
|
+
]);
|
|
5254
5270
|
var ObjectReference = import_v36.z.object({
|
|
5255
5271
|
object_type: import_v36.z.enum([
|
|
5256
5272
|
"project_logs",
|
|
@@ -5272,6 +5288,7 @@ var TraceScope = import_v36.z.object({
|
|
|
5272
5288
|
});
|
|
5273
5289
|
var OnlineScoreConfig = import_v36.z.union([
|
|
5274
5290
|
import_v36.z.object({
|
|
5291
|
+
status: AutomationStatus.optional(),
|
|
5275
5292
|
sampling_rate: import_v36.z.number().gte(0).lte(1),
|
|
5276
5293
|
scorers: import_v36.z.array(SavedFunctionId),
|
|
5277
5294
|
btql_filter: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
@@ -5352,6 +5369,72 @@ var Project = import_v36.z.object({
|
|
|
5352
5369
|
user_id: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
5353
5370
|
settings: ProjectSettings.optional()
|
|
5354
5371
|
});
|
|
5372
|
+
var WindowedAutomationConfig = import_v36.z.object({
|
|
5373
|
+
event_type: import_v36.z.literal("windowed"),
|
|
5374
|
+
product_origin: import_v36.z.union([import_v36.z.literal("patterns"), import_v36.z.null()]).optional(),
|
|
5375
|
+
status: AutomationStatus.optional(),
|
|
5376
|
+
threshold: import_v36.z.object({
|
|
5377
|
+
calculation: import_v36.z.object({
|
|
5378
|
+
type: import_v36.z.literal("btql"),
|
|
5379
|
+
btql_query: import_v36.z.string().min(1),
|
|
5380
|
+
output: import_v36.z.object({
|
|
5381
|
+
type: import_v36.z.literal("scalar"),
|
|
5382
|
+
value_column: import_v36.z.string().min(1)
|
|
5383
|
+
})
|
|
5384
|
+
}),
|
|
5385
|
+
policy: import_v36.z.object({
|
|
5386
|
+
condition: import_v36.z.object({
|
|
5387
|
+
type: import_v36.z.literal("threshold"),
|
|
5388
|
+
operator: import_v36.z.enum(["lt", "lte", "gt", "gte", "eq", "neq"]),
|
|
5389
|
+
threshold: import_v36.z.number()
|
|
5390
|
+
}),
|
|
5391
|
+
pending_seconds: import_v36.z.number().int().gte(0).lte(2592e3),
|
|
5392
|
+
no_data_behavior: import_v36.z.enum(["keep_last", "resolve", "alert"]),
|
|
5393
|
+
renotify_interval_seconds: import_v36.z.union([import_v36.z.number(), import_v36.z.null()]).optional(),
|
|
5394
|
+
notify_on_recovery: import_v36.z.boolean().optional().default(true)
|
|
5395
|
+
})
|
|
5396
|
+
}).optional(),
|
|
5397
|
+
window: import_v36.z.object({
|
|
5398
|
+
window_seconds: import_v36.z.number().int().gte(1).lte(2592e3),
|
|
5399
|
+
schedule: import_v36.z.union([
|
|
5400
|
+
import_v36.z.object({
|
|
5401
|
+
type: import_v36.z.literal("interval"),
|
|
5402
|
+
evaluation_interval_seconds: import_v36.z.number().int().gte(1).lte(2592e3)
|
|
5403
|
+
}),
|
|
5404
|
+
import_v36.z.object({
|
|
5405
|
+
type: import_v36.z.literal("cron"),
|
|
5406
|
+
cron_expression: import_v36.z.string().min(1),
|
|
5407
|
+
timezone: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional()
|
|
5408
|
+
})
|
|
5409
|
+
]),
|
|
5410
|
+
evaluation_delay_seconds: import_v36.z.number().int().gte(0).lte(2592e3)
|
|
5411
|
+
}),
|
|
5412
|
+
loop: import_v36.z.object({
|
|
5413
|
+
prompt: import_v36.z.string().min(1).max(1e4),
|
|
5414
|
+
include_trigger_input: import_v36.z.boolean().optional().default(false),
|
|
5415
|
+
agent_slug: import_v36.z.string().min(1),
|
|
5416
|
+
auto_approve_tools: import_v36.z.array(import_v36.z.string().min(1)).optional().default([]),
|
|
5417
|
+
harness: import_v36.z.enum(["native", "codex", "claude-code"]).optional(),
|
|
5418
|
+
model: import_v36.z.string().min(1).optional(),
|
|
5419
|
+
reasoning_effort: import_v36.z.enum(["none", "minimal", "low", "medium", "high", "xhigh", "max"]).optional()
|
|
5420
|
+
}).optional(),
|
|
5421
|
+
actions: import_v36.z.array(
|
|
5422
|
+
import_v36.z.union([
|
|
5423
|
+
import_v36.z.object({
|
|
5424
|
+
type: import_v36.z.literal("webhook"),
|
|
5425
|
+
url: import_v36.z.string(),
|
|
5426
|
+
formatting_prompt: import_v36.z.string().min(1).max(1e4).optional()
|
|
5427
|
+
}),
|
|
5428
|
+
import_v36.z.object({
|
|
5429
|
+
type: import_v36.z.literal("slack"),
|
|
5430
|
+
workspace_id: import_v36.z.string(),
|
|
5431
|
+
channel: import_v36.z.string(),
|
|
5432
|
+
message_template: import_v36.z.string().optional(),
|
|
5433
|
+
formatting_prompt: import_v36.z.string().min(1).max(1e4).optional()
|
|
5434
|
+
})
|
|
5435
|
+
])
|
|
5436
|
+
).max(20).optional().default([])
|
|
5437
|
+
});
|
|
5355
5438
|
var TopicAutomationFacetModel = import_v36.z.union([
|
|
5356
5439
|
import_v36.z.enum(["brain-facet-latest", "brain-facet-1", "brain-facet-2"]),
|
|
5357
5440
|
import_v36.z.null()
|
|
@@ -5393,7 +5476,8 @@ var TopicDigestAutomationConfig = import_v36.z.object({
|
|
|
5393
5476
|
type: import_v36.z.literal("slack"),
|
|
5394
5477
|
workspace_id: import_v36.z.string(),
|
|
5395
5478
|
channel: import_v36.z.string(),
|
|
5396
|
-
message_template: import_v36.z.string().optional()
|
|
5479
|
+
message_template: import_v36.z.string().optional(),
|
|
5480
|
+
formatting_prompt: import_v36.z.string().min(1).max(1e4).optional()
|
|
5397
5481
|
}),
|
|
5398
5482
|
topic_map_function_ids: import_v36.z.array(import_v36.z.string()).max(10).optional()
|
|
5399
5483
|
});
|
|
@@ -5407,15 +5491,21 @@ var ProjectAutomation = import_v36.z.object({
|
|
|
5407
5491
|
config: import_v36.z.union([
|
|
5408
5492
|
import_v36.z.object({
|
|
5409
5493
|
event_type: import_v36.z.literal("logs"),
|
|
5494
|
+
status: AutomationStatus.optional(),
|
|
5410
5495
|
btql_filter: import_v36.z.string(),
|
|
5411
5496
|
interval_seconds: import_v36.z.number().gte(1).lte(2592e3),
|
|
5412
5497
|
action: import_v36.z.union([
|
|
5413
|
-
import_v36.z.object({
|
|
5498
|
+
import_v36.z.object({
|
|
5499
|
+
type: import_v36.z.literal("webhook"),
|
|
5500
|
+
url: import_v36.z.string(),
|
|
5501
|
+
formatting_prompt: import_v36.z.string().min(1).max(1e4).optional()
|
|
5502
|
+
}),
|
|
5414
5503
|
import_v36.z.object({
|
|
5415
5504
|
type: import_v36.z.literal("slack"),
|
|
5416
5505
|
workspace_id: import_v36.z.string(),
|
|
5417
5506
|
channel: import_v36.z.string(),
|
|
5418
|
-
message_template: import_v36.z.string().optional()
|
|
5507
|
+
message_template: import_v36.z.string().optional(),
|
|
5508
|
+
formatting_prompt: import_v36.z.string().min(1).max(1e4).optional()
|
|
5419
5509
|
})
|
|
5420
5510
|
])
|
|
5421
5511
|
}),
|
|
@@ -5466,21 +5556,38 @@ var ProjectAutomation = import_v36.z.object({
|
|
|
5466
5556
|
}),
|
|
5467
5557
|
import_v36.z.object({
|
|
5468
5558
|
event_type: import_v36.z.literal("environment_update"),
|
|
5559
|
+
status: AutomationStatus.optional(),
|
|
5469
5560
|
environment_filter: import_v36.z.array(import_v36.z.string()).optional(),
|
|
5470
5561
|
action: import_v36.z.union([
|
|
5471
|
-
import_v36.z.object({
|
|
5562
|
+
import_v36.z.object({
|
|
5563
|
+
type: import_v36.z.literal("webhook"),
|
|
5564
|
+
url: import_v36.z.string(),
|
|
5565
|
+
formatting_prompt: import_v36.z.string().min(1).max(1e4).optional()
|
|
5566
|
+
}),
|
|
5472
5567
|
import_v36.z.object({
|
|
5473
5568
|
type: import_v36.z.literal("slack"),
|
|
5474
5569
|
workspace_id: import_v36.z.string(),
|
|
5475
5570
|
channel: import_v36.z.string(),
|
|
5476
|
-
message_template: import_v36.z.string().optional()
|
|
5571
|
+
message_template: import_v36.z.string().optional(),
|
|
5572
|
+
formatting_prompt: import_v36.z.string().min(1).max(1e4).optional()
|
|
5477
5573
|
})
|
|
5478
5574
|
])
|
|
5479
5575
|
}),
|
|
5576
|
+
WindowedAutomationConfig,
|
|
5480
5577
|
TopicAutomationConfig,
|
|
5481
5578
|
TopicDigestAutomationConfig
|
|
5482
5579
|
])
|
|
5483
5580
|
});
|
|
5581
|
+
var ProjectGroup = import_v36.z.object({
|
|
5582
|
+
id: import_v36.z.string().uuid(),
|
|
5583
|
+
org_id: import_v36.z.string().uuid(),
|
|
5584
|
+
user_id: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
5585
|
+
created: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
5586
|
+
name: import_v36.z.string(),
|
|
5587
|
+
description: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
5588
|
+
deleted_at: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
5589
|
+
member_projects: import_v36.z.array(import_v36.z.string().uuid()).max(1e4)
|
|
5590
|
+
});
|
|
5484
5591
|
var ProjectLogsEvent = import_v36.z.object({
|
|
5485
5592
|
id: import_v36.z.string(),
|
|
5486
5593
|
_xact_id: import_v36.z.string(),
|
|
@@ -5698,7 +5805,8 @@ var RunEval = import_v36.z.object({
|
|
|
5698
5805
|
dataset_environment: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
5699
5806
|
_internal_btql: import_v36.z.union([import_v36.z.object({}).partial().passthrough(), import_v36.z.null()]).optional()
|
|
5700
5807
|
}),
|
|
5701
|
-
import_v36.z.object({ data: import_v36.z.array(import_v36.z.unknown()) })
|
|
5808
|
+
import_v36.z.object({ data: import_v36.z.array(import_v36.z.unknown()) }),
|
|
5809
|
+
import_v36.z.object({ experiment_name: import_v36.z.string() })
|
|
5702
5810
|
]),
|
|
5703
5811
|
name: import_v36.z.string().optional(),
|
|
5704
5812
|
parameters: import_v36.z.object({}).partial().passthrough().optional(),
|
|
@@ -5903,7 +6011,9 @@ var View = import_v36.z.object({
|
|
|
5903
6011
|
"for_review_datasets"
|
|
5904
6012
|
]),
|
|
5905
6013
|
name: import_v36.z.string(),
|
|
6014
|
+
description: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
5906
6015
|
created: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
6016
|
+
updated_at: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
5907
6017
|
view_data: ViewData.optional(),
|
|
5908
6018
|
options: ViewOptions.optional(),
|
|
5909
6019
|
user_id: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
@@ -6428,10 +6538,10 @@ var DiskCache = class {
|
|
|
6428
6538
|
return;
|
|
6429
6539
|
}
|
|
6430
6540
|
const stats = await Promise.all(
|
|
6431
|
-
paths.map(async (
|
|
6432
|
-
const stat2 = await isomorph_default.stat(
|
|
6541
|
+
paths.map(async (path9) => {
|
|
6542
|
+
const stat2 = await isomorph_default.stat(path9);
|
|
6433
6543
|
return {
|
|
6434
|
-
path:
|
|
6544
|
+
path: path9,
|
|
6435
6545
|
mtime: stat2.mtime.getTime()
|
|
6436
6546
|
};
|
|
6437
6547
|
})
|
|
@@ -7028,12 +7138,13 @@ var INSTRUMENTATION_NAMES = {
|
|
|
7028
7138
|
OPENROUTER: "openrouter",
|
|
7029
7139
|
OPENROUTER_AGENT: "openrouter-agent",
|
|
7030
7140
|
PI_CODING_AGENT: "pi-coding-agent",
|
|
7031
|
-
STRANDS_AGENT_SDK: "strands-agent-sdk"
|
|
7141
|
+
STRANDS_AGENT_SDK: "strands-agent-sdk",
|
|
7142
|
+
VOYAGEAI: "voyageai"
|
|
7032
7143
|
};
|
|
7033
7144
|
var INTERNAL_SPAN_INSTRUMENTATION_NAME = /* @__PURE__ */ Symbol.for(
|
|
7034
7145
|
"braintrust.spanInstrumentationName"
|
|
7035
7146
|
);
|
|
7036
|
-
var SDK_VERSION = true ? "3.
|
|
7147
|
+
var SDK_VERSION = true ? "3.28.0" : "0.0.0";
|
|
7037
7148
|
function withSpanInstrumentationName(args, instrumentationName) {
|
|
7038
7149
|
return {
|
|
7039
7150
|
...args,
|
|
@@ -7228,6 +7339,9 @@ function applyMaskingToField(maskingFunction, data, fieldName) {
|
|
|
7228
7339
|
var INITIAL_SPAN_WRITE_AS_MERGE = /* @__PURE__ */ Symbol(
|
|
7229
7340
|
"braintrust.initial-span-write-as-merge"
|
|
7230
7341
|
);
|
|
7342
|
+
var RESUME_SPAN_WITHOUT_INITIAL_WRITE = /* @__PURE__ */ Symbol(
|
|
7343
|
+
"braintrust.resume-span-without-initial-write"
|
|
7344
|
+
);
|
|
7231
7345
|
var INTERNAL_SPAN_CONTEXT = /* @__PURE__ */ Symbol("braintrust.internal-span-context");
|
|
7232
7346
|
var BRAINTRUST_CURRENT_SPAN_STORE = /* @__PURE__ */ Symbol.for(
|
|
7233
7347
|
"braintrust.currentSpanStore"
|
|
@@ -7793,9 +7907,9 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
7793
7907
|
this.headers["Authorization"] = `Bearer ${this.token}`;
|
|
7794
7908
|
}
|
|
7795
7909
|
}
|
|
7796
|
-
async get(
|
|
7910
|
+
async get(path9, params = void 0, config3) {
|
|
7797
7911
|
const { headers, ...rest } = config3 || {};
|
|
7798
|
-
const url = new URL(_urljoin(this.base_url,
|
|
7912
|
+
const url = new URL(_urljoin(this.base_url, path9));
|
|
7799
7913
|
url.search = new URLSearchParams(
|
|
7800
7914
|
params ? Object.entries(params).filter(([_, v]) => v !== void 0).flatMap(
|
|
7801
7915
|
([k, v]) => v !== void 0 ? typeof v === "string" ? [[k, v]] : v.map((x) => [k, x]) : []
|
|
@@ -7816,7 +7930,7 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
7816
7930
|
})
|
|
7817
7931
|
);
|
|
7818
7932
|
}
|
|
7819
|
-
async post(
|
|
7933
|
+
async post(path9, params, config3, retries = 0) {
|
|
7820
7934
|
const { headers, ...rest } = config3 || {};
|
|
7821
7935
|
const this_fetch = this.fetch;
|
|
7822
7936
|
const this_base_url = this.base_url;
|
|
@@ -7825,7 +7939,7 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
7825
7939
|
for (let i = 0; i < tries; i++) {
|
|
7826
7940
|
try {
|
|
7827
7941
|
return await checkResponse(
|
|
7828
|
-
await this_fetch(_urljoin(this_base_url,
|
|
7942
|
+
await this_fetch(_urljoin(this_base_url, path9), {
|
|
7829
7943
|
method: "POST",
|
|
7830
7944
|
headers: {
|
|
7831
7945
|
Accept: "application/json",
|
|
@@ -7846,7 +7960,7 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
7846
7960
|
throw error2;
|
|
7847
7961
|
}
|
|
7848
7962
|
debugLogger.debug(
|
|
7849
|
-
`Retrying API request ${
|
|
7963
|
+
`Retrying API request ${path9} after ${formatHTTPError(error2)}`
|
|
7850
7964
|
);
|
|
7851
7965
|
const sleepTimeMs = HTTP_RETRY_BASE_SLEEP_TIME_S * 1e3 * 2 ** i + Math.random() * HTTP_RETRY_JITTER_MS;
|
|
7852
7966
|
debugLogger.info(
|
|
@@ -10611,7 +10725,7 @@ var ObjectFetcher = class {
|
|
|
10611
10725
|
const objectId = await this.id;
|
|
10612
10726
|
const batchLimit = batchSize ?? DEFAULT_FETCH_BATCH_SIZE;
|
|
10613
10727
|
const internalLimit = getInternalBtqlLimit(this._internal_btql);
|
|
10614
|
-
|
|
10728
|
+
let remainingLimit = internalLimit;
|
|
10615
10729
|
const internalBtqlWithoutReservedQueryKeys = Object.fromEntries(
|
|
10616
10730
|
Object.entries(this._internal_btql ?? {}).filter(
|
|
10617
10731
|
([key]) => key !== "cursor" && key !== "limit" && key !== "select" && key !== "from"
|
|
@@ -10620,6 +10734,10 @@ var ObjectFetcher = class {
|
|
|
10620
10734
|
let cursor = void 0;
|
|
10621
10735
|
let iterations = 0;
|
|
10622
10736
|
while (true) {
|
|
10737
|
+
if (remainingLimit !== void 0 && remainingLimit <= 0) {
|
|
10738
|
+
return;
|
|
10739
|
+
}
|
|
10740
|
+
const limit = remainingLimit === void 0 ? batchLimit : Math.min(batchLimit, remainingLimit);
|
|
10623
10741
|
const resp = await state.apiConn().post(
|
|
10624
10742
|
`btql`,
|
|
10625
10743
|
{
|
|
@@ -10659,7 +10777,14 @@ var ObjectFetcher = class {
|
|
|
10659
10777
|
const respJson = await resp.json();
|
|
10660
10778
|
const mutate = this.mutateRecord;
|
|
10661
10779
|
for (const record of respJson.data ?? []) {
|
|
10662
|
-
|
|
10780
|
+
if (remainingLimit !== void 0 && remainingLimit <= 0) {
|
|
10781
|
+
return;
|
|
10782
|
+
}
|
|
10783
|
+
const mutatedRecord = mutate ? mutate(record) : record;
|
|
10784
|
+
if (remainingLimit !== void 0) {
|
|
10785
|
+
remainingLimit--;
|
|
10786
|
+
}
|
|
10787
|
+
yield mutatedRecord;
|
|
10663
10788
|
}
|
|
10664
10789
|
if (!respJson.cursor) {
|
|
10665
10790
|
break;
|
|
@@ -11179,7 +11304,9 @@ var SpanImpl = class _SpanImpl {
|
|
|
11179
11304
|
this._rootSpanId = resolvedIds.rootSpanId;
|
|
11180
11305
|
this._spanParents = resolvedIds.spanParents;
|
|
11181
11306
|
this.isMerge = args[INITIAL_SPAN_WRITE_AS_MERGE] === true;
|
|
11182
|
-
|
|
11307
|
+
if (!args[RESUME_SPAN_WITHOUT_INITIAL_WRITE]) {
|
|
11308
|
+
this.logInternal({ event, internalData });
|
|
11309
|
+
}
|
|
11183
11310
|
this.isMerge = true;
|
|
11184
11311
|
}
|
|
11185
11312
|
getParentInfo() {
|
|
@@ -13931,8 +14058,8 @@ function validateParametersWithJsonSchema(parameters, schema) {
|
|
|
13931
14058
|
const validate = ajv.compile(schema);
|
|
13932
14059
|
if (!validate(parameters)) {
|
|
13933
14060
|
const errorMessages = validate.errors?.map((err) => {
|
|
13934
|
-
const
|
|
13935
|
-
return `${
|
|
14061
|
+
const path9 = err.instancePath || "root";
|
|
14062
|
+
return `${path9}: ${err.message}`;
|
|
13936
14063
|
}).join(", ");
|
|
13937
14064
|
throw Error(`Invalid parameters: ${errorMessages}`);
|
|
13938
14065
|
}
|
|
@@ -13961,6 +14088,9 @@ function rehydrateRemoteParameters(parameters, schema) {
|
|
|
13961
14088
|
}
|
|
13962
14089
|
|
|
13963
14090
|
// src/framework.ts
|
|
14091
|
+
function BaseExperiment(options = {}) {
|
|
14092
|
+
return { _type: "BaseExperiment", ...options };
|
|
14093
|
+
}
|
|
13964
14094
|
var EvalResultWithSummary = class {
|
|
13965
14095
|
constructor(summary, results) {
|
|
13966
14096
|
this.summary = summary;
|
|
@@ -14021,6 +14151,26 @@ async function getExperimentParametersRef(parameters) {
|
|
|
14021
14151
|
version: resolvedParameters.version
|
|
14022
14152
|
};
|
|
14023
14153
|
}
|
|
14154
|
+
async function _internalInitEvaluatorExperiment(projectName, evaluator, data, options = {}) {
|
|
14155
|
+
if (options.disabled) return null;
|
|
14156
|
+
const { baseExperiment } = callEvaluatorData(data);
|
|
14157
|
+
const parameters = await getExperimentParametersRef(evaluator.parameters);
|
|
14158
|
+
return initExperiment(evaluator.state, {
|
|
14159
|
+
...evaluator.projectId ? { projectId: evaluator.projectId } : { project: projectName },
|
|
14160
|
+
experiment: options.experimentName ?? evaluator.experimentName,
|
|
14161
|
+
description: evaluator.description,
|
|
14162
|
+
metadata: evaluator.metadata,
|
|
14163
|
+
tags: evaluator.tags,
|
|
14164
|
+
isPublic: evaluator.isPublic,
|
|
14165
|
+
update: options.update ?? evaluator.update,
|
|
14166
|
+
baseExperiment: evaluator.baseExperimentName ?? baseExperiment,
|
|
14167
|
+
baseExperimentId: evaluator.baseExperimentId,
|
|
14168
|
+
gitMetadataSettings: evaluator.gitMetadataSettings,
|
|
14169
|
+
repoInfo: evaluator.repoInfo,
|
|
14170
|
+
dataset: Dataset2.isDataset(data) ? data : void 0,
|
|
14171
|
+
parameters
|
|
14172
|
+
});
|
|
14173
|
+
}
|
|
14024
14174
|
function callEvaluatorData(data) {
|
|
14025
14175
|
const dataResult = typeof data === "function" ? data() : data;
|
|
14026
14176
|
let baseExperiment = void 0;
|
|
@@ -14038,6 +14188,48 @@ function isAsyncIterable2(value) {
|
|
|
14038
14188
|
function isIterable(value) {
|
|
14039
14189
|
return typeof value === "object" && value !== null && Symbol.iterator in value && typeof value[Symbol.iterator] === "function";
|
|
14040
14190
|
}
|
|
14191
|
+
async function _internalResolveEvaluatorData(evaluator, experiment) {
|
|
14192
|
+
if (typeof evaluator.data === "string") {
|
|
14193
|
+
throw new Error("Unimplemented: string data paths");
|
|
14194
|
+
}
|
|
14195
|
+
let dataResult = typeof evaluator.data === "function" ? evaluator.data() : evaluator.data;
|
|
14196
|
+
if ("_type" in dataResult) {
|
|
14197
|
+
if (dataResult._type !== "BaseExperiment") {
|
|
14198
|
+
throw new Error("Invalid _type");
|
|
14199
|
+
}
|
|
14200
|
+
if (!experiment) {
|
|
14201
|
+
throw new Error(
|
|
14202
|
+
"Cannot use BaseExperiment() without connecting to Braintrust (you most likely set --no-send-logs)"
|
|
14203
|
+
);
|
|
14204
|
+
}
|
|
14205
|
+
let name = dataResult.name;
|
|
14206
|
+
if (isEmpty2(name)) {
|
|
14207
|
+
const baseExperiment = await experiment.fetchBaseExperiment();
|
|
14208
|
+
if (!baseExperiment) {
|
|
14209
|
+
throw new Error("BaseExperiment() failed to fetch base experiment");
|
|
14210
|
+
}
|
|
14211
|
+
name = baseExperiment.name;
|
|
14212
|
+
}
|
|
14213
|
+
dataResult = initExperiment(evaluator.state, {
|
|
14214
|
+
...evaluator.projectId ? { projectId: evaluator.projectId } : { project: evaluator.projectName },
|
|
14215
|
+
experiment: name,
|
|
14216
|
+
open: true
|
|
14217
|
+
}).asDataset();
|
|
14218
|
+
}
|
|
14219
|
+
const resolvedDataResult = dataResult instanceof Promise ? await dataResult : dataResult;
|
|
14220
|
+
if (isAsyncIterable2(resolvedDataResult)) {
|
|
14221
|
+
return resolvedDataResult;
|
|
14222
|
+
}
|
|
14223
|
+
if (Array.isArray(resolvedDataResult) || isIterable(resolvedDataResult)) {
|
|
14224
|
+
const iterable = resolvedDataResult;
|
|
14225
|
+
return (async function* () {
|
|
14226
|
+
for (const datum of iterable) yield datum;
|
|
14227
|
+
})();
|
|
14228
|
+
}
|
|
14229
|
+
throw new Error(
|
|
14230
|
+
"Evaluator data must be an array, iterable, or async iterable"
|
|
14231
|
+
);
|
|
14232
|
+
}
|
|
14041
14233
|
globalThis._evals = {
|
|
14042
14234
|
functions: [],
|
|
14043
14235
|
prompts: [],
|
|
@@ -14084,25 +14276,13 @@ async function Eval(name, evaluator, reporterOrOpts) {
|
|
|
14084
14276
|
}
|
|
14085
14277
|
const resolvedReporter = options.reporter || defaultReporter;
|
|
14086
14278
|
try {
|
|
14087
|
-
const { data
|
|
14088
|
-
|
|
14089
|
-
|
|
14090
|
-
|
|
14091
|
-
|
|
14092
|
-
|
|
14093
|
-
|
|
14094
|
-
description: evaluator.description,
|
|
14095
|
-
metadata: evaluator.metadata,
|
|
14096
|
-
tags: evaluator.tags,
|
|
14097
|
-
isPublic: evaluator.isPublic,
|
|
14098
|
-
update: evaluator.update,
|
|
14099
|
-
baseExperiment: evaluator.baseExperimentName ?? defaultBaseExperiment,
|
|
14100
|
-
baseExperimentId: evaluator.baseExperimentId,
|
|
14101
|
-
gitMetadataSettings: evaluator.gitMetadataSettings,
|
|
14102
|
-
repoInfo: evaluator.repoInfo,
|
|
14103
|
-
dataset: Dataset2.isDataset(data) ? data : void 0,
|
|
14104
|
-
parameters
|
|
14105
|
-
});
|
|
14279
|
+
const { data } = callEvaluatorData(evaluator.data);
|
|
14280
|
+
const experiment = await _internalInitEvaluatorExperiment(
|
|
14281
|
+
name,
|
|
14282
|
+
evaluator,
|
|
14283
|
+
data,
|
|
14284
|
+
{ disabled: Boolean(options.parent || options.noSendLogs) }
|
|
14285
|
+
);
|
|
14106
14286
|
if (experiment && typeof process !== "undefined" && globalThis.BRAINTRUST_CONTEXT_MANAGER !== void 0) {
|
|
14107
14287
|
await experiment._waitForId();
|
|
14108
14288
|
}
|
|
@@ -14184,21 +14364,21 @@ function parseFilters(filters) {
|
|
|
14184
14364
|
if (equalsIdx === -1) {
|
|
14185
14365
|
throw new Error(`Invalid filter ${f}`);
|
|
14186
14366
|
}
|
|
14187
|
-
const [
|
|
14367
|
+
const [path9, value] = [f.slice(0, equalsIdx), f.slice(equalsIdx + 1)];
|
|
14188
14368
|
let deserializedValue = deserializePlainStringAsJSON2(value).value;
|
|
14189
14369
|
if (typeof deserializedValue !== "string") {
|
|
14190
14370
|
deserializedValue = value;
|
|
14191
14371
|
}
|
|
14192
14372
|
result.push({
|
|
14193
|
-
path:
|
|
14373
|
+
path: path9.split("."),
|
|
14194
14374
|
pattern: new RegExp(deserializedValue)
|
|
14195
14375
|
});
|
|
14196
14376
|
}
|
|
14197
14377
|
return result;
|
|
14198
14378
|
}
|
|
14199
14379
|
function evaluateFilter(object, filter2) {
|
|
14200
|
-
const { path:
|
|
14201
|
-
const key =
|
|
14380
|
+
const { path: path9, pattern } = filter2;
|
|
14381
|
+
const key = path9.reduce(
|
|
14202
14382
|
(acc, p) => typeof acc === "object" && acc !== null ? (
|
|
14203
14383
|
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
14204
14384
|
acc[p]
|
|
@@ -14216,19 +14396,74 @@ function scorerName(scorer, scorer_idx) {
|
|
|
14216
14396
|
function classifierName(classifier, classifier_idx) {
|
|
14217
14397
|
return classifier.name || `classifier_${classifier_idx}`;
|
|
14218
14398
|
}
|
|
14399
|
+
async function _internalRunEvaluatorTask(task, datum, trialIndex, parameters, span, reportProgress = () => void 0) {
|
|
14400
|
+
const metadata = {
|
|
14401
|
+
..."metadata" in datum ? datum.metadata : {}
|
|
14402
|
+
};
|
|
14403
|
+
const hooks = {
|
|
14404
|
+
meta(value) {
|
|
14405
|
+
Object.assign(metadata, value);
|
|
14406
|
+
},
|
|
14407
|
+
metadata,
|
|
14408
|
+
expected: "expected" in datum ? datum.expected : void 0,
|
|
14409
|
+
span,
|
|
14410
|
+
parameters,
|
|
14411
|
+
reportProgress,
|
|
14412
|
+
trialIndex,
|
|
14413
|
+
tags: [...datum.tags ?? []]
|
|
14414
|
+
};
|
|
14415
|
+
const output = await task(datum.input, hooks);
|
|
14416
|
+
span.log({ output });
|
|
14417
|
+
return {
|
|
14418
|
+
output,
|
|
14419
|
+
metadata: hooks.metadata,
|
|
14420
|
+
tags: hooks.tags ?? []
|
|
14421
|
+
};
|
|
14422
|
+
}
|
|
14219
14423
|
function buildSpanMetadata(results) {
|
|
14220
|
-
return results.length === 1 ? results[0].metadata :
|
|
14221
|
-
(
|
|
14222
|
-
{}
|
|
14424
|
+
return results.length === 1 ? results[0].metadata : Object.fromEntries(
|
|
14425
|
+
results.map((result) => [result.name, result.metadata])
|
|
14223
14426
|
);
|
|
14224
14427
|
}
|
|
14225
14428
|
function buildSpanScores(results) {
|
|
14226
|
-
const scoresRecord =
|
|
14227
|
-
(
|
|
14228
|
-
{}
|
|
14429
|
+
const scoresRecord = Object.fromEntries(
|
|
14430
|
+
results.map((result) => [result.name, result.score])
|
|
14229
14431
|
);
|
|
14230
14432
|
return { resultMetadata: buildSpanMetadata(results), scoresRecord };
|
|
14231
14433
|
}
|
|
14434
|
+
function _internalPrepareEvaluatorScore(scoreValue, name) {
|
|
14435
|
+
if (scoreValue === null) return { results: null };
|
|
14436
|
+
if (Array.isArray(scoreValue)) {
|
|
14437
|
+
for (const score of scoreValue) {
|
|
14438
|
+
if (!(typeof score === "object" && !isEmpty2(score))) {
|
|
14439
|
+
throw new Error(
|
|
14440
|
+
`When returning an array of scores, each score must be a non-empty object. Got: ${JSON.stringify(score)}`
|
|
14441
|
+
);
|
|
14442
|
+
}
|
|
14443
|
+
}
|
|
14444
|
+
}
|
|
14445
|
+
let results;
|
|
14446
|
+
if (Array.isArray(scoreValue)) {
|
|
14447
|
+
results = scoreValue;
|
|
14448
|
+
} else if (typeof scoreValue === "object" && !isEmpty2(scoreValue)) {
|
|
14449
|
+
results = [scoreValue];
|
|
14450
|
+
} else {
|
|
14451
|
+
results = [{ name, score: scoreValue }];
|
|
14452
|
+
}
|
|
14453
|
+
const { resultMetadata, scoresRecord } = buildSpanScores(results);
|
|
14454
|
+
const fields = (score) => {
|
|
14455
|
+
const { metadata: _metadata, name: _name, ...rest } = score;
|
|
14456
|
+
return rest;
|
|
14457
|
+
};
|
|
14458
|
+
return {
|
|
14459
|
+
results,
|
|
14460
|
+
output: results.length === 1 ? fields(results[0]) : Object.fromEntries(
|
|
14461
|
+
results.map((score) => [score.name ?? name, fields(score)])
|
|
14462
|
+
),
|
|
14463
|
+
metadata: resultMetadata,
|
|
14464
|
+
scores: scoresRecord
|
|
14465
|
+
};
|
|
14466
|
+
}
|
|
14232
14467
|
async function runInScorerSpan(rootSpan, spanName, spanType, propagatedEvent, eventInput, fn) {
|
|
14233
14468
|
try {
|
|
14234
14469
|
const value = await rootSpan.traced(fn, {
|
|
@@ -14273,6 +14508,27 @@ function toClassificationItem(c) {
|
|
|
14273
14508
|
...c.metadata !== void 0 ? { metadata: c.metadata } : {}
|
|
14274
14509
|
};
|
|
14275
14510
|
}
|
|
14511
|
+
function _internalPrepareEvaluatorClassification(value, name) {
|
|
14512
|
+
if (value === null) return { results: null };
|
|
14513
|
+
const results = (Array.isArray(value) ? value : [value]).map(
|
|
14514
|
+
(result) => validateClassificationResult(result, name)
|
|
14515
|
+
);
|
|
14516
|
+
const classifications = /* @__PURE__ */ Object.create(null);
|
|
14517
|
+
for (const result of results) {
|
|
14518
|
+
(classifications[result.name] ??= []).push(toClassificationItem(result));
|
|
14519
|
+
}
|
|
14520
|
+
return {
|
|
14521
|
+
results,
|
|
14522
|
+
output: results.length === 1 ? toClassificationItem(results[0]) : Object.fromEntries(
|
|
14523
|
+
results.map((result) => [
|
|
14524
|
+
result.name,
|
|
14525
|
+
toClassificationItem(result)
|
|
14526
|
+
])
|
|
14527
|
+
),
|
|
14528
|
+
metadata: buildSpanMetadata(results),
|
|
14529
|
+
classifications
|
|
14530
|
+
};
|
|
14531
|
+
}
|
|
14276
14532
|
function logScoringFailures(kind, failures, metadata, rootSpan, state) {
|
|
14277
14533
|
if (!failures.length) return [];
|
|
14278
14534
|
const errorMap = Object.fromEntries(
|
|
@@ -14311,54 +14567,14 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
14311
14567
|
(evaluator.state ?? _internalGetGlobalState())?.spanCache?.start();
|
|
14312
14568
|
}
|
|
14313
14569
|
try {
|
|
14314
|
-
if (typeof evaluator.data === "string") {
|
|
14315
|
-
throw new Error("Unimplemented: string data paths");
|
|
14316
|
-
}
|
|
14317
|
-
let dataResult = typeof evaluator.data === "function" ? evaluator.data() : evaluator.data;
|
|
14318
14570
|
parameters = await validateParameters(
|
|
14319
14571
|
parameters ?? {},
|
|
14320
14572
|
evaluator.parameters
|
|
14321
14573
|
);
|
|
14322
|
-
|
|
14323
|
-
|
|
14324
|
-
|
|
14325
|
-
|
|
14326
|
-
if (!experiment) {
|
|
14327
|
-
throw new Error(
|
|
14328
|
-
"Cannot use BaseExperiment() without connecting to Braintrust (you most likely set --no-send-logs)"
|
|
14329
|
-
);
|
|
14330
|
-
}
|
|
14331
|
-
let name = dataResult.name;
|
|
14332
|
-
if (isEmpty2(name)) {
|
|
14333
|
-
const baseExperiment = await experiment.fetchBaseExperiment();
|
|
14334
|
-
if (!baseExperiment) {
|
|
14335
|
-
throw new Error("BaseExperiment() failed to fetch base experiment");
|
|
14336
|
-
}
|
|
14337
|
-
name = baseExperiment.name;
|
|
14338
|
-
}
|
|
14339
|
-
dataResult = initExperiment(evaluator.state, {
|
|
14340
|
-
...evaluator.projectId ? { projectId: evaluator.projectId } : { project: evaluator.projectName },
|
|
14341
|
-
experiment: name,
|
|
14342
|
-
open: true
|
|
14343
|
-
}).asDataset();
|
|
14344
|
-
}
|
|
14345
|
-
const resolvedDataResult = dataResult instanceof Promise ? await dataResult : dataResult;
|
|
14346
|
-
const dataIterable = (() => {
|
|
14347
|
-
if (isAsyncIterable2(resolvedDataResult)) {
|
|
14348
|
-
return resolvedDataResult;
|
|
14349
|
-
}
|
|
14350
|
-
if (Array.isArray(resolvedDataResult) || isIterable(resolvedDataResult)) {
|
|
14351
|
-
const iterable = resolvedDataResult;
|
|
14352
|
-
return (async function* () {
|
|
14353
|
-
for (const datum of iterable) {
|
|
14354
|
-
yield datum;
|
|
14355
|
-
}
|
|
14356
|
-
})();
|
|
14357
|
-
}
|
|
14358
|
-
throw new Error(
|
|
14359
|
-
"Evaluator data must be an array, iterable, or async iterable"
|
|
14360
|
-
);
|
|
14361
|
-
})();
|
|
14574
|
+
const dataIterable = await _internalResolveEvaluatorData(
|
|
14575
|
+
evaluator,
|
|
14576
|
+
experiment
|
|
14577
|
+
);
|
|
14362
14578
|
progressReporter.start(evaluator.evalName, 0);
|
|
14363
14579
|
const experimentIdPromise = experiment ? (async () => {
|
|
14364
14580
|
try {
|
|
@@ -14427,57 +14643,45 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
14427
14643
|
ensureSpansFlushed,
|
|
14428
14644
|
state
|
|
14429
14645
|
}) : void 0;
|
|
14430
|
-
let metadata = {
|
|
14431
|
-
..."metadata" in datum ? datum.metadata : {}
|
|
14432
|
-
};
|
|
14646
|
+
let metadata = {};
|
|
14433
14647
|
const expected = "expected" in datum ? datum.expected : void 0;
|
|
14434
14648
|
let output = void 0;
|
|
14435
14649
|
let error2 = void 0;
|
|
14436
|
-
let tags = [
|
|
14437
|
-
const scores =
|
|
14438
|
-
const classifications =
|
|
14650
|
+
let tags = [];
|
|
14651
|
+
const scores = /* @__PURE__ */ Object.create(null);
|
|
14652
|
+
const classifications = /* @__PURE__ */ Object.create(null);
|
|
14439
14653
|
const scorerNames = (evaluator.scores ?? []).map(scorerName);
|
|
14440
14654
|
const classifierNames = (evaluator.classifiers ?? []).map(
|
|
14441
14655
|
classifierName
|
|
14442
14656
|
);
|
|
14443
14657
|
let unhandledScores = scorerNames;
|
|
14444
14658
|
try {
|
|
14445
|
-
const
|
|
14446
|
-
|
|
14447
|
-
|
|
14448
|
-
|
|
14449
|
-
|
|
14450
|
-
|
|
14451
|
-
|
|
14452
|
-
|
|
14453
|
-
|
|
14454
|
-
|
|
14455
|
-
|
|
14456
|
-
|
|
14457
|
-
|
|
14458
|
-
|
|
14459
|
-
|
|
14460
|
-
object_type: "task"
|
|
14461
|
-
});
|
|
14462
|
-
},
|
|
14463
|
-
trialIndex,
|
|
14464
|
-
tags
|
|
14465
|
-
};
|
|
14466
|
-
const outputResult = evaluator.task(datum.input, hooksForTask);
|
|
14467
|
-
if (outputResult instanceof Promise) {
|
|
14468
|
-
output = await outputResult;
|
|
14469
|
-
} else {
|
|
14470
|
-
output = outputResult;
|
|
14659
|
+
const taskResult = await rootSpan.traced(
|
|
14660
|
+
(span) => _internalRunEvaluatorTask(
|
|
14661
|
+
evaluator.task,
|
|
14662
|
+
datum,
|
|
14663
|
+
trialIndex,
|
|
14664
|
+
parameters ?? {},
|
|
14665
|
+
span,
|
|
14666
|
+
(event) => {
|
|
14667
|
+
stream?.({
|
|
14668
|
+
...event,
|
|
14669
|
+
id: rootSpan.id,
|
|
14670
|
+
origin: baseEvent.event?.origin,
|
|
14671
|
+
name: evaluator.evalName,
|
|
14672
|
+
object_type: "task"
|
|
14673
|
+
});
|
|
14471
14674
|
}
|
|
14472
|
-
|
|
14473
|
-
span.log({ output });
|
|
14474
|
-
},
|
|
14675
|
+
),
|
|
14475
14676
|
{
|
|
14476
14677
|
name: "task",
|
|
14477
14678
|
spanAttributes: { type: "task" /* TASK */ },
|
|
14478
14679
|
event: { input: datum.input }
|
|
14479
14680
|
}
|
|
14480
14681
|
);
|
|
14682
|
+
output = taskResult.output;
|
|
14683
|
+
metadata = taskResult.metadata;
|
|
14684
|
+
tags = taskResult.tags;
|
|
14481
14685
|
if (tags.length) {
|
|
14482
14686
|
rootSpan.log({ output, metadata, expected, tags });
|
|
14483
14687
|
} else {
|
|
@@ -14487,20 +14691,18 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
14487
14691
|
await rootSpan.flush();
|
|
14488
14692
|
}
|
|
14489
14693
|
const scoringArgs = {
|
|
14694
|
+
id: datum.id,
|
|
14490
14695
|
input: datum.input,
|
|
14491
14696
|
expected: "expected" in datum ? datum.expected : void 0,
|
|
14492
14697
|
metadata,
|
|
14493
14698
|
output,
|
|
14699
|
+
tags,
|
|
14494
14700
|
trace
|
|
14495
14701
|
};
|
|
14496
14702
|
const { trace: _trace, ...scoringArgsForLogging } = scoringArgs;
|
|
14497
14703
|
const propagatedEvent = makeScorerPropagatedEvent(
|
|
14498
14704
|
await rootSpan.export()
|
|
14499
14705
|
);
|
|
14500
|
-
const getOtherFields = (s) => {
|
|
14501
|
-
const { metadata: _metadata, name: _name, ...rest } = s;
|
|
14502
|
-
return rest;
|
|
14503
|
-
};
|
|
14504
14706
|
const [scoreResults, classificationResults] = await Promise.all([
|
|
14505
14707
|
Promise.all(
|
|
14506
14708
|
(evaluator.scores ?? []).map(
|
|
@@ -14514,35 +14716,17 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
14514
14716
|
const scoreValue = await Promise.resolve(
|
|
14515
14717
|
score(scoringArgs)
|
|
14516
14718
|
);
|
|
14517
|
-
|
|
14518
|
-
|
|
14519
|
-
|
|
14520
|
-
if (!(typeof s === "object" && !isEmpty2(s))) {
|
|
14521
|
-
throw new Error(
|
|
14522
|
-
`When returning an array of scores, each score must be a non-empty object. Got: ${JSON.stringify(s)}`
|
|
14523
|
-
);
|
|
14524
|
-
}
|
|
14525
|
-
}
|
|
14526
|
-
}
|
|
14527
|
-
const results = Array.isArray(scoreValue) ? scoreValue : typeof scoreValue === "object" && !isEmpty2(scoreValue) ? [scoreValue] : [
|
|
14528
|
-
{
|
|
14529
|
-
name: scorerNames[score_idx],
|
|
14530
|
-
score: scoreValue
|
|
14531
|
-
}
|
|
14532
|
-
];
|
|
14533
|
-
const { resultMetadata, scoresRecord } = buildSpanScores(results);
|
|
14534
|
-
const resultOutput = results.length === 1 ? getOtherFields(results[0]) : results.reduce(
|
|
14535
|
-
(prev, s) => mergeDicts(prev, {
|
|
14536
|
-
[s.name]: getOtherFields(s)
|
|
14537
|
-
}),
|
|
14538
|
-
{}
|
|
14719
|
+
const prepared = _internalPrepareEvaluatorScore(
|
|
14720
|
+
scoreValue,
|
|
14721
|
+
scorerNames[score_idx]
|
|
14539
14722
|
);
|
|
14723
|
+
if (prepared.results === null) return null;
|
|
14540
14724
|
span.log({
|
|
14541
|
-
output:
|
|
14542
|
-
metadata:
|
|
14543
|
-
scores:
|
|
14725
|
+
output: prepared.output,
|
|
14726
|
+
metadata: prepared.metadata,
|
|
14727
|
+
scores: prepared.scores
|
|
14544
14728
|
});
|
|
14545
|
-
return results;
|
|
14729
|
+
return prepared.results;
|
|
14546
14730
|
}
|
|
14547
14731
|
)
|
|
14548
14732
|
)
|
|
@@ -14559,24 +14743,16 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
14559
14743
|
const classifierValue = await Promise.resolve(
|
|
14560
14744
|
classifier(scoringArgs)
|
|
14561
14745
|
);
|
|
14562
|
-
|
|
14563
|
-
|
|
14564
|
-
|
|
14565
|
-
result,
|
|
14566
|
-
classifierNames[idx]
|
|
14567
|
-
)
|
|
14568
|
-
);
|
|
14569
|
-
const resultOutput = rawResults.length === 1 ? toClassificationItem(rawResults[0]) : rawResults.reduce(
|
|
14570
|
-
(prev, r) => mergeDicts(prev, {
|
|
14571
|
-
[r.name]: toClassificationItem(r)
|
|
14572
|
-
}),
|
|
14573
|
-
{}
|
|
14746
|
+
const prepared = _internalPrepareEvaluatorClassification(
|
|
14747
|
+
classifierValue,
|
|
14748
|
+
classifierNames[idx]
|
|
14574
14749
|
);
|
|
14750
|
+
if (prepared.results === null) return null;
|
|
14575
14751
|
span.log({
|
|
14576
|
-
output:
|
|
14577
|
-
metadata:
|
|
14752
|
+
output: prepared.output,
|
|
14753
|
+
metadata: prepared.metadata
|
|
14578
14754
|
});
|
|
14579
|
-
return
|
|
14755
|
+
return prepared.results;
|
|
14580
14756
|
}
|
|
14581
14757
|
)
|
|
14582
14758
|
)
|
|
@@ -14806,7 +14982,7 @@ function accumulateScores(accumulator, scores) {
|
|
|
14806
14982
|
}
|
|
14807
14983
|
}
|
|
14808
14984
|
function ensureScoreAccumulator(results) {
|
|
14809
|
-
const accumulator =
|
|
14985
|
+
const accumulator = /* @__PURE__ */ Object.create(null);
|
|
14810
14986
|
for (const result of results) {
|
|
14811
14987
|
accumulateScores(accumulator, result.scores);
|
|
14812
14988
|
}
|
|
@@ -15117,7 +15293,7 @@ var fancyReporter = {
|
|
|
15117
15293
|
|
|
15118
15294
|
// src/node/config.ts
|
|
15119
15295
|
var import_node_async_hooks = require("node:async_hooks");
|
|
15120
|
-
var
|
|
15296
|
+
var path2 = __toESM(require("node:path"));
|
|
15121
15297
|
var fs = __toESM(require("node:fs/promises"));
|
|
15122
15298
|
var os = __toESM(require("node:os"));
|
|
15123
15299
|
var fsSync = __toESM(require("node:fs"));
|
|
@@ -15126,34 +15302,93 @@ var import_node_util3 = require("node:util");
|
|
|
15126
15302
|
var zlib = __toESM(require("node:zlib"));
|
|
15127
15303
|
var dotenv = __toESM(require("dotenv"));
|
|
15128
15304
|
|
|
15129
|
-
// src/
|
|
15130
|
-
var
|
|
15131
|
-
var
|
|
15132
|
-
|
|
15133
|
-
|
|
15134
|
-
|
|
15135
|
-
|
|
15136
|
-
|
|
15137
|
-
|
|
15138
|
-
|
|
15305
|
+
// src/git-command.ts
|
|
15306
|
+
var import_node_child_process = require("node:child_process");
|
|
15307
|
+
var import_node_fs = require("node:fs");
|
|
15308
|
+
var import_promises = require("node:fs/promises");
|
|
15309
|
+
var path = __toESM(require("node:path"));
|
|
15310
|
+
var GIT_EXECUTABLE_NAMES = process.platform === "win32" ? ["git.exe", "git"] : ["git"];
|
|
15311
|
+
var GIT_MAX_BUFFER_BYTES = 10 * 1024 * 1024;
|
|
15312
|
+
var gitExecutablePromise;
|
|
15313
|
+
function executableSearchPath() {
|
|
15314
|
+
return Object.entries(process.env).find(
|
|
15315
|
+
([name]) => name.toUpperCase() === "PATH"
|
|
15316
|
+
)?.[1];
|
|
15317
|
+
}
|
|
15318
|
+
async function findGitExecutable(searchPath = executableSearchPath()) {
|
|
15319
|
+
if (!searchPath) {
|
|
15320
|
+
return void 0;
|
|
15321
|
+
}
|
|
15322
|
+
for (const rawSearchDir of searchPath.split(path.delimiter)) {
|
|
15323
|
+
const searchDir = rawSearchDir.trim().replace(/^"(.*)"$/, "$1");
|
|
15324
|
+
if (!path.isAbsolute(searchDir)) {
|
|
15325
|
+
continue;
|
|
15139
15326
|
}
|
|
15140
|
-
|
|
15141
|
-
|
|
15327
|
+
for (const executableName of GIT_EXECUTABLE_NAMES) {
|
|
15328
|
+
const candidate = path.join(searchDir, executableName);
|
|
15329
|
+
try {
|
|
15330
|
+
await (0, import_promises.access)(
|
|
15331
|
+
candidate,
|
|
15332
|
+
process.platform === "win32" ? import_node_fs.constants.F_OK : import_node_fs.constants.X_OK
|
|
15333
|
+
);
|
|
15334
|
+
return candidate;
|
|
15335
|
+
} catch {
|
|
15336
|
+
}
|
|
15337
|
+
}
|
|
15338
|
+
}
|
|
15339
|
+
return void 0;
|
|
15340
|
+
}
|
|
15341
|
+
async function resolveGitExecutable() {
|
|
15342
|
+
gitExecutablePromise ??= findGitExecutable();
|
|
15343
|
+
return await gitExecutablePromise;
|
|
15344
|
+
}
|
|
15345
|
+
async function runGitCommand(args, options = {}) {
|
|
15346
|
+
const executable = await resolveGitExecutable();
|
|
15347
|
+
if (!executable) {
|
|
15348
|
+
throw new Error("Could not find a git executable on PATH");
|
|
15142
15349
|
}
|
|
15350
|
+
return await new Promise((resolve2, reject2) => {
|
|
15351
|
+
(0, import_node_child_process.execFile)(
|
|
15352
|
+
executable,
|
|
15353
|
+
args,
|
|
15354
|
+
{
|
|
15355
|
+
cwd: options.cwd,
|
|
15356
|
+
encoding: "utf8",
|
|
15357
|
+
maxBuffer: GIT_MAX_BUFFER_BYTES
|
|
15358
|
+
},
|
|
15359
|
+
(error2, stdout) => {
|
|
15360
|
+
if (error2) {
|
|
15361
|
+
reject2(error2);
|
|
15362
|
+
} else {
|
|
15363
|
+
resolve2(stdout);
|
|
15364
|
+
}
|
|
15365
|
+
}
|
|
15366
|
+
);
|
|
15367
|
+
});
|
|
15143
15368
|
}
|
|
15369
|
+
|
|
15370
|
+
// src/gitutil.ts
|
|
15371
|
+
var COMMON_BASE_BRANCHES = ["main", "master", "develop"];
|
|
15144
15372
|
var _baseBranch = null;
|
|
15145
15373
|
async function getBaseBranch(remote = void 0) {
|
|
15146
15374
|
if (_baseBranch === null) {
|
|
15147
|
-
const
|
|
15148
|
-
if (
|
|
15375
|
+
const repoPath = await currentRepoPath();
|
|
15376
|
+
if (!repoPath) {
|
|
15149
15377
|
throw new Error("Not in a git repo");
|
|
15150
15378
|
}
|
|
15151
|
-
const
|
|
15379
|
+
const runGit = async (args) => await runGitCommand(args, { cwd: repoPath });
|
|
15380
|
+
const remoteName = remote ?? (await runGit(["remote"])).trim().split(/\r?\n/)[0];
|
|
15152
15381
|
if (!remoteName) {
|
|
15153
15382
|
throw new Error("No remote found");
|
|
15154
15383
|
}
|
|
15155
15384
|
let branch = null;
|
|
15156
|
-
const repoBranches = new Set(
|
|
15385
|
+
const repoBranches = new Set(
|
|
15386
|
+
(await runGit([
|
|
15387
|
+
"for-each-ref",
|
|
15388
|
+
"--format=%(refname:short)",
|
|
15389
|
+
"refs/heads/"
|
|
15390
|
+
])).trim().split(/\r?\n/)
|
|
15391
|
+
);
|
|
15157
15392
|
const matchingBaseBranches = COMMON_BASE_BRANCHES.filter(
|
|
15158
15393
|
(b) => repoBranches.has(b)
|
|
15159
15394
|
);
|
|
@@ -15161,7 +15396,7 @@ async function getBaseBranch(remote = void 0) {
|
|
|
15161
15396
|
branch = matchingBaseBranches[0];
|
|
15162
15397
|
} else {
|
|
15163
15398
|
try {
|
|
15164
|
-
const remoteInfo = await
|
|
15399
|
+
const remoteInfo = await runGit(["remote", "show", remoteName]);
|
|
15165
15400
|
if (!remoteInfo) {
|
|
15166
15401
|
throw new Error(`Could not find remote ${remoteName}`);
|
|
15167
15402
|
}
|
|
@@ -15179,27 +15414,28 @@ async function getBaseBranch(remote = void 0) {
|
|
|
15179
15414
|
return _baseBranch;
|
|
15180
15415
|
}
|
|
15181
15416
|
async function getBaseBranchAncestor(remote = void 0) {
|
|
15182
|
-
const
|
|
15183
|
-
if (
|
|
15417
|
+
const repoPath = await currentRepoPath();
|
|
15418
|
+
if (!repoPath) {
|
|
15184
15419
|
throw new Error("Not in a git repo");
|
|
15185
15420
|
}
|
|
15186
15421
|
const { remote: remoteName, branch: baseBranch } = await getBaseBranch(remote);
|
|
15187
|
-
const isDirty = (await
|
|
15422
|
+
const isDirty = (await runGitCommand(["diff", "--name-only"], {
|
|
15423
|
+
cwd: repoPath
|
|
15424
|
+
})).trim().length > 0;
|
|
15188
15425
|
const head = isDirty ? "HEAD" : "HEAD^";
|
|
15189
15426
|
try {
|
|
15190
|
-
const ancestor = await
|
|
15191
|
-
"merge-base",
|
|
15192
|
-
|
|
15193
|
-
|
|
15194
|
-
]);
|
|
15427
|
+
const ancestor = await runGitCommand(
|
|
15428
|
+
["merge-base", head, `${remoteName}/${baseBranch}`],
|
|
15429
|
+
{ cwd: repoPath }
|
|
15430
|
+
);
|
|
15195
15431
|
return ancestor.trim();
|
|
15196
15432
|
} catch {
|
|
15197
15433
|
return void 0;
|
|
15198
15434
|
}
|
|
15199
15435
|
}
|
|
15200
15436
|
async function getPastNAncestors(n = 1e3, remote = void 0) {
|
|
15201
|
-
const
|
|
15202
|
-
if (
|
|
15437
|
+
const repoPath = await currentRepoPath();
|
|
15438
|
+
if (!repoPath) {
|
|
15203
15439
|
return [];
|
|
15204
15440
|
}
|
|
15205
15441
|
let ancestor = void 0;
|
|
@@ -15214,8 +15450,10 @@ async function getPastNAncestors(n = 1e3, remote = void 0) {
|
|
|
15214
15450
|
if (!ancestor) {
|
|
15215
15451
|
return [];
|
|
15216
15452
|
}
|
|
15217
|
-
const commits = await
|
|
15218
|
-
|
|
15453
|
+
const commits = (await runGitCommand(["rev-list", `--max-count=${n}`, `${ancestor}..HEAD`], {
|
|
15454
|
+
cwd: repoPath
|
|
15455
|
+
})).trim();
|
|
15456
|
+
return commits ? commits.split(/\r?\n/).slice(0, n) : [];
|
|
15219
15457
|
}
|
|
15220
15458
|
async function attempt(fn) {
|
|
15221
15459
|
try {
|
|
@@ -15246,9 +15484,14 @@ async function getRepoInfo(settings) {
|
|
|
15246
15484
|
});
|
|
15247
15485
|
return sanitized;
|
|
15248
15486
|
}
|
|
15487
|
+
async function currentRepoPath() {
|
|
15488
|
+
return await attempt(
|
|
15489
|
+
async () => (await runGitCommand(["rev-parse", "--show-toplevel"])).trim()
|
|
15490
|
+
);
|
|
15491
|
+
}
|
|
15249
15492
|
async function repoInfo() {
|
|
15250
|
-
const
|
|
15251
|
-
if (
|
|
15493
|
+
const repoPath = await currentRepoPath();
|
|
15494
|
+
if (!repoPath) {
|
|
15252
15495
|
return void 0;
|
|
15253
15496
|
}
|
|
15254
15497
|
let commit = void 0;
|
|
@@ -15259,29 +15502,32 @@ async function repoInfo() {
|
|
|
15259
15502
|
let tag = void 0;
|
|
15260
15503
|
let branch = void 0;
|
|
15261
15504
|
let git_diff = void 0;
|
|
15262
|
-
const
|
|
15263
|
-
|
|
15505
|
+
const runGit = async (args) => await runGitCommand(args, { cwd: repoPath });
|
|
15506
|
+
const dirty = (await runGit(["diff", "--name-only"])).trim().length > 0;
|
|
15507
|
+
commit = await attempt(
|
|
15508
|
+
async () => (await runGit(["rev-parse", "HEAD"])).trim()
|
|
15509
|
+
);
|
|
15264
15510
|
commit_message = await attempt(
|
|
15265
|
-
async () => (await
|
|
15511
|
+
async () => (await runGit(["log", "-1", "--pretty=%B"])).trim()
|
|
15266
15512
|
);
|
|
15267
15513
|
commit_time = await attempt(
|
|
15268
|
-
async () => (await
|
|
15514
|
+
async () => (await runGit(["log", "-1", "--pretty=%cI"])).trim()
|
|
15269
15515
|
);
|
|
15270
15516
|
author_name = await attempt(
|
|
15271
|
-
async () => (await
|
|
15517
|
+
async () => (await runGit(["log", "-1", "--pretty=%aN"])).trim()
|
|
15272
15518
|
);
|
|
15273
15519
|
author_email = await attempt(
|
|
15274
|
-
async () => (await
|
|
15520
|
+
async () => (await runGit(["log", "-1", "--pretty=%aE"])).trim()
|
|
15275
15521
|
);
|
|
15276
15522
|
tag = await attempt(
|
|
15277
|
-
async () => (await
|
|
15523
|
+
async () => (await runGit(["describe", "--tags", "--exact-match", "--always"])).trim()
|
|
15278
15524
|
);
|
|
15279
15525
|
branch = await attempt(
|
|
15280
|
-
async () => (await
|
|
15526
|
+
async () => (await runGit(["rev-parse", "--abbrev-ref", "HEAD"])).trim()
|
|
15281
15527
|
);
|
|
15282
15528
|
if (dirty) {
|
|
15283
15529
|
git_diff = await attempt(
|
|
15284
|
-
async () => truncateToByteLimit(await
|
|
15530
|
+
async () => truncateToByteLimit(await runGit(["diff", "--no-ext-diff", "HEAD"]))
|
|
15285
15531
|
);
|
|
15286
15532
|
}
|
|
15287
15533
|
return {
|
|
@@ -16747,6 +16993,22 @@ function processInputAttachments(input) {
|
|
|
16747
16993
|
};
|
|
16748
16994
|
}
|
|
16749
16995
|
}
|
|
16996
|
+
const voyageBase64Key = node.type === "image_base64" ? Object.hasOwn(node, "imageBase64") ? "imageBase64" : "image_base64" : node.type === "video_base64" ? Object.hasOwn(node, "videoBase64") ? "videoBase64" : "video_base64" : void 0;
|
|
16997
|
+
const voyageBase64Value = voyageBase64Key ? node[voyageBase64Key] : void 0;
|
|
16998
|
+
if (voyageBase64Key && typeof voyageBase64Value === "string" && voyageBase64Value.startsWith("data:")) {
|
|
16999
|
+
const mediaType = inferMediaTypeFromDataUrl(
|
|
17000
|
+
voyageBase64Value,
|
|
17001
|
+
node.type === "video_base64" ? "video/mp4" : "image/png"
|
|
17002
|
+
);
|
|
17003
|
+
const filename = `${node.type === "video_base64" ? "video" : "image"}.${getExtensionFromMediaType(mediaType)}`;
|
|
17004
|
+
const attachment = toAttachment(voyageBase64Value, mediaType, filename);
|
|
17005
|
+
if (attachment) {
|
|
17006
|
+
return {
|
|
17007
|
+
...node,
|
|
17008
|
+
[voyageBase64Key]: attachment
|
|
17009
|
+
};
|
|
17010
|
+
}
|
|
17011
|
+
}
|
|
16750
17012
|
if (node.type === "file" && node.file && typeof node.file === "object" && typeof node.file.file_data === "string" && node.file.file_data.startsWith("data:")) {
|
|
16751
17013
|
const mediaType = inferMediaTypeFromDataUrl(
|
|
16752
17014
|
node.file.file_data,
|
|
@@ -18094,12 +18356,30 @@ function logInstrumentationError(context2, error2) {
|
|
|
18094
18356
|
|
|
18095
18357
|
// src/wrappers/anthropic-tokens-util.ts
|
|
18096
18358
|
function finalizeAnthropicTokens(metrics) {
|
|
18097
|
-
const
|
|
18098
|
-
|
|
18359
|
+
const hasSplitCacheCreationTokens = metrics.prompt_cache_creation_5m_tokens !== void 0 || metrics.prompt_cache_creation_1h_tokens !== void 0;
|
|
18360
|
+
const splitCacheCreationTokens = (metrics.prompt_cache_creation_5m_tokens || 0) + (metrics.prompt_cache_creation_1h_tokens || 0);
|
|
18361
|
+
const aggregateCacheCreationTokens = metrics.prompt_cache_creation_tokens || 0;
|
|
18362
|
+
const effectiveCacheCreationTokens = Math.max(
|
|
18363
|
+
aggregateCacheCreationTokens,
|
|
18364
|
+
splitCacheCreationTokens
|
|
18365
|
+
);
|
|
18366
|
+
const prompt_tokens = (metrics.prompt_tokens || 0) + (metrics.prompt_cached_tokens || 0) + effectiveCacheCreationTokens;
|
|
18367
|
+
const finalized = {
|
|
18099
18368
|
...metrics,
|
|
18100
18369
|
prompt_tokens,
|
|
18101
18370
|
tokens: prompt_tokens + (metrics.completion_tokens || 0)
|
|
18102
18371
|
};
|
|
18372
|
+
if (hasSplitCacheCreationTokens && splitCacheCreationTokens >= aggregateCacheCreationTokens) {
|
|
18373
|
+
delete finalized.prompt_cache_creation_tokens;
|
|
18374
|
+
}
|
|
18375
|
+
return finalized;
|
|
18376
|
+
}
|
|
18377
|
+
function toNumericMetrics(metrics) {
|
|
18378
|
+
return Object.fromEntries(
|
|
18379
|
+
Object.entries(metrics).filter(
|
|
18380
|
+
(entry) => entry[1] !== void 0
|
|
18381
|
+
)
|
|
18382
|
+
);
|
|
18103
18383
|
}
|
|
18104
18384
|
function extractAnthropicCacheTokens(cacheReadTokens = 0, cacheCreationTokens = 0) {
|
|
18105
18385
|
const cacheTokens = {};
|
|
@@ -18692,6 +18972,18 @@ function parseMetricsFromUsage2(usage) {
|
|
|
18692
18972
|
saveIfExistsTo("output_tokens", "completion_tokens");
|
|
18693
18973
|
saveIfExistsTo("cache_read_input_tokens", "prompt_cached_tokens");
|
|
18694
18974
|
saveIfExistsTo("cache_creation_input_tokens", "prompt_cache_creation_tokens");
|
|
18975
|
+
if (isObject(usage.cache_creation)) {
|
|
18976
|
+
const cacheCreation = usage.cache_creation;
|
|
18977
|
+
for (const [source, target] of [
|
|
18978
|
+
["ephemeral_5m_input_tokens", "prompt_cache_creation_5m_tokens"],
|
|
18979
|
+
["ephemeral_1h_input_tokens", "prompt_cache_creation_1h_tokens"]
|
|
18980
|
+
]) {
|
|
18981
|
+
const value = cacheCreation[source];
|
|
18982
|
+
if (typeof value === "number") {
|
|
18983
|
+
metrics[target] = value;
|
|
18984
|
+
}
|
|
18985
|
+
}
|
|
18986
|
+
}
|
|
18695
18987
|
if (isObject(usage.server_tool_use)) {
|
|
18696
18988
|
for (const [name, value] of Object.entries(usage.server_tool_use)) {
|
|
18697
18989
|
if (typeof value === "number") {
|
|
@@ -21283,7 +21575,7 @@ function resolveDenyOutputPaths(event, defaultDenyOutputPaths) {
|
|
|
21283
21575
|
return defaultDenyOutputPaths;
|
|
21284
21576
|
}
|
|
21285
21577
|
const runtimeDenyOutputPaths = firstArgument2[RUNTIME_DENY_OUTPUT_PATHS];
|
|
21286
|
-
if (Array.isArray(runtimeDenyOutputPaths) && runtimeDenyOutputPaths.every((
|
|
21578
|
+
if (Array.isArray(runtimeDenyOutputPaths) && runtimeDenyOutputPaths.every((path9) => typeof path9 === "string")) {
|
|
21287
21579
|
return runtimeDenyOutputPaths;
|
|
21288
21580
|
}
|
|
21289
21581
|
return defaultDenyOutputPaths;
|
|
@@ -23045,11 +23337,11 @@ function processAISDKOutput(output, denyOutputPaths) {
|
|
|
23045
23337
|
if (!output) return output;
|
|
23046
23338
|
const merged = extractSerializableOutputFields(output);
|
|
23047
23339
|
const deleteOutputPaths = denyOutputPaths.filter(
|
|
23048
|
-
(
|
|
23340
|
+
(path9) => path9.toLowerCase().endsWith("headers")
|
|
23049
23341
|
);
|
|
23050
23342
|
const sanitized = omit(merged, denyOutputPaths, deleteOutputPaths);
|
|
23051
|
-
for (const
|
|
23052
|
-
const stack = [{ obj: sanitized, keys: parsePath(
|
|
23343
|
+
for (const path9 of TRANSPORT_PAYLOAD_ROOT_PATHS) {
|
|
23344
|
+
const stack = [{ obj: sanitized, keys: parsePath(path9) }];
|
|
23053
23345
|
while (stack.length > 0) {
|
|
23054
23346
|
const entry = stack.pop();
|
|
23055
23347
|
if (!entry || entry.keys.length === 0) {
|
|
@@ -23488,11 +23780,11 @@ function firstNumber2(...values) {
|
|
|
23488
23780
|
function deepCopy(obj) {
|
|
23489
23781
|
return JSON.parse(JSON.stringify(obj));
|
|
23490
23782
|
}
|
|
23491
|
-
function parsePath(
|
|
23783
|
+
function parsePath(path9) {
|
|
23492
23784
|
const keys = [];
|
|
23493
23785
|
let current = "";
|
|
23494
|
-
for (let i = 0; i <
|
|
23495
|
-
const char =
|
|
23786
|
+
for (let i = 0; i < path9.length; i++) {
|
|
23787
|
+
const char = path9[i];
|
|
23496
23788
|
if (char === ".") {
|
|
23497
23789
|
if (current) {
|
|
23498
23790
|
keys.push(current);
|
|
@@ -23505,8 +23797,8 @@ function parsePath(path8) {
|
|
|
23505
23797
|
}
|
|
23506
23798
|
let bracketContent = "";
|
|
23507
23799
|
i++;
|
|
23508
|
-
while (i <
|
|
23509
|
-
bracketContent +=
|
|
23800
|
+
while (i < path9.length && path9[i] !== "]") {
|
|
23801
|
+
bracketContent += path9[i];
|
|
23510
23802
|
i++;
|
|
23511
23803
|
}
|
|
23512
23804
|
if (bracketContent === "") {
|
|
@@ -23561,9 +23853,9 @@ function omitAtPath(obj, keys, deleteLeaf = false) {
|
|
|
23561
23853
|
function omit(obj, paths, deletePaths = []) {
|
|
23562
23854
|
const result = deepCopy(obj);
|
|
23563
23855
|
const deletePathSet = new Set(deletePaths);
|
|
23564
|
-
for (const
|
|
23565
|
-
const keys = parsePath(
|
|
23566
|
-
omitAtPath(result, keys, deletePathSet.has(
|
|
23856
|
+
for (const path9 of paths) {
|
|
23857
|
+
const keys = parsePath(path9);
|
|
23858
|
+
omitAtPath(result, keys, deletePathSet.has(path9));
|
|
23567
23859
|
}
|
|
23568
23860
|
return result;
|
|
23569
23861
|
}
|
|
@@ -23990,37 +24282,97 @@ function seedTaskToolUseIdMapping(taskIdToToolUseId, message) {
|
|
|
23990
24282
|
taskIdToToolUseId.set(message.task_id, message.tool_use_id);
|
|
23991
24283
|
}
|
|
23992
24284
|
}
|
|
23993
|
-
function
|
|
23994
|
-
|
|
23995
|
-
|
|
23996
|
-
|
|
23997
|
-
usage = message.message?.usage;
|
|
23998
|
-
} else if (message.type === "result") {
|
|
23999
|
-
usage = message.usage;
|
|
24000
|
-
}
|
|
24285
|
+
function tokenCount(value) {
|
|
24286
|
+
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value) && value >= 0 ? value : void 0;
|
|
24287
|
+
}
|
|
24288
|
+
function copyUsage(usage) {
|
|
24001
24289
|
if (!usage || typeof usage !== "object") {
|
|
24002
|
-
return
|
|
24290
|
+
return void 0;
|
|
24291
|
+
}
|
|
24292
|
+
const copy = {};
|
|
24293
|
+
for (const key of [
|
|
24294
|
+
"input_tokens",
|
|
24295
|
+
"output_tokens",
|
|
24296
|
+
"cache_read_input_tokens",
|
|
24297
|
+
"cache_creation_input_tokens"
|
|
24298
|
+
]) {
|
|
24299
|
+
const value = tokenCount(Reflect.get(usage, key));
|
|
24300
|
+
if (value !== void 0) {
|
|
24301
|
+
copy[key] = value;
|
|
24302
|
+
}
|
|
24303
|
+
}
|
|
24304
|
+
const cacheCreation = Reflect.get(usage, "cache_creation");
|
|
24305
|
+
if (cacheCreation && typeof cacheCreation === "object") {
|
|
24306
|
+
const cacheCreationCopy = {};
|
|
24307
|
+
for (const key of [
|
|
24308
|
+
"ephemeral_5m_input_tokens",
|
|
24309
|
+
"ephemeral_1h_input_tokens"
|
|
24310
|
+
]) {
|
|
24311
|
+
const value = tokenCount(Reflect.get(cacheCreation, key));
|
|
24312
|
+
if (value !== void 0) {
|
|
24313
|
+
cacheCreationCopy[key] = value;
|
|
24314
|
+
}
|
|
24315
|
+
}
|
|
24316
|
+
if (Object.keys(cacheCreationCopy).length > 0) {
|
|
24317
|
+
copy.cache_creation = cacheCreationCopy;
|
|
24318
|
+
}
|
|
24319
|
+
}
|
|
24320
|
+
return Object.keys(copy).length > 0 ? copy : void 0;
|
|
24321
|
+
}
|
|
24322
|
+
function mergeUsage(base, override) {
|
|
24323
|
+
if (!base || !override) {
|
|
24324
|
+
return override ?? base;
|
|
24325
|
+
}
|
|
24326
|
+
const cacheCreation = base.cache_creation || override.cache_creation ? { ...base.cache_creation, ...override.cache_creation } : void 0;
|
|
24327
|
+
return {
|
|
24328
|
+
...base,
|
|
24329
|
+
...override,
|
|
24330
|
+
...cacheCreation && { cache_creation: cacheCreation }
|
|
24331
|
+
};
|
|
24332
|
+
}
|
|
24333
|
+
function extractUsage(usage, includeOutput) {
|
|
24334
|
+
const metrics = {};
|
|
24335
|
+
if (!usage) {
|
|
24336
|
+
return {};
|
|
24003
24337
|
}
|
|
24004
24338
|
const inputTokens = getNumberProperty(usage, "input_tokens");
|
|
24005
24339
|
if (inputTokens !== void 0) {
|
|
24006
24340
|
metrics.prompt_tokens = inputTokens;
|
|
24007
24341
|
}
|
|
24008
|
-
|
|
24009
|
-
|
|
24010
|
-
|
|
24342
|
+
if (includeOutput) {
|
|
24343
|
+
const outputTokens = getNumberProperty(usage, "output_tokens");
|
|
24344
|
+
if (outputTokens !== void 0) {
|
|
24345
|
+
metrics.completion_tokens = outputTokens;
|
|
24346
|
+
}
|
|
24011
24347
|
}
|
|
24012
24348
|
const cacheReadTokens = getNumberProperty(usage, "cache_read_input_tokens") || 0;
|
|
24013
24349
|
const cacheCreationTokens = getNumberProperty(usage, "cache_creation_input_tokens") || 0;
|
|
24014
|
-
|
|
24015
|
-
|
|
24016
|
-
|
|
24017
|
-
|
|
24018
|
-
|
|
24350
|
+
Object.assign(
|
|
24351
|
+
metrics,
|
|
24352
|
+
extractAnthropicCacheTokens(cacheReadTokens, cacheCreationTokens)
|
|
24353
|
+
);
|
|
24354
|
+
const cacheCreation5mTokens = getNumberProperty(
|
|
24355
|
+
usage.cache_creation,
|
|
24356
|
+
"ephemeral_5m_input_tokens"
|
|
24357
|
+
);
|
|
24358
|
+
const cacheCreation1hTokens = getNumberProperty(
|
|
24359
|
+
usage.cache_creation,
|
|
24360
|
+
"ephemeral_1h_input_tokens"
|
|
24361
|
+
);
|
|
24362
|
+
if (cacheCreation5mTokens !== void 0) {
|
|
24363
|
+
metrics.prompt_cache_creation_5m_tokens = cacheCreation5mTokens;
|
|
24019
24364
|
}
|
|
24020
|
-
if (
|
|
24021
|
-
|
|
24365
|
+
if (cacheCreation1hTokens !== void 0) {
|
|
24366
|
+
metrics.prompt_cache_creation_1h_tokens = cacheCreation1hTokens;
|
|
24022
24367
|
}
|
|
24023
|
-
|
|
24368
|
+
if (Object.keys(metrics).length === 0) {
|
|
24369
|
+
return {};
|
|
24370
|
+
}
|
|
24371
|
+
const finalized = finalizeAnthropicTokens(metrics);
|
|
24372
|
+
if (metrics.completion_tokens === void 0) {
|
|
24373
|
+
delete finalized.tokens;
|
|
24374
|
+
}
|
|
24375
|
+
return toNumericMetrics(finalized);
|
|
24024
24376
|
}
|
|
24025
24377
|
function buildLLMInput(promptMessages, conversationHistory) {
|
|
24026
24378
|
const inputParts = [...promptMessages, ...conversationHistory];
|
|
@@ -24054,16 +24406,16 @@ function buildRootPromptMessages(prompt, capturedPromptMessages) {
|
|
|
24054
24406
|
function formatCapturedMessages(messages) {
|
|
24055
24407
|
return messages.length > 0 ? messages : [];
|
|
24056
24408
|
}
|
|
24057
|
-
async function createLLMSpanForMessages(messages, promptMessages, conversationHistory, options, startTime, parentSpan, existingSpan) {
|
|
24409
|
+
async function createLLMSpanForMessages(messages, promptMessages, conversationHistory, options, startTime, parentSpan, usage, hasFinalOutputUsage, existingSpan) {
|
|
24058
24410
|
if (messages.length === 0) {
|
|
24059
24411
|
return void 0;
|
|
24060
24412
|
}
|
|
24061
24413
|
const lastMessage = messages[messages.length - 1];
|
|
24062
|
-
if (lastMessage.type !== "assistant"
|
|
24414
|
+
if (lastMessage.type !== "assistant") {
|
|
24063
24415
|
return void 0;
|
|
24064
24416
|
}
|
|
24065
|
-
const model = lastMessage.message
|
|
24066
|
-
const
|
|
24417
|
+
const model = lastMessage.message?.model || options.model;
|
|
24418
|
+
const metrics = options.includePartialMessages ? extractUsage(usage, hasFinalOutputUsage) : {};
|
|
24067
24419
|
const input = buildLLMInput(promptMessages, conversationHistory);
|
|
24068
24420
|
const outputs = messages.map(
|
|
24069
24421
|
(m) => m.message?.content && m.message?.role ? { content: m.message.content, role: m.message.role } : void 0
|
|
@@ -24085,8 +24437,8 @@ async function createLLMSpanForMessages(messages, promptMessages, conversationHi
|
|
|
24085
24437
|
);
|
|
24086
24438
|
span.log({
|
|
24087
24439
|
input,
|
|
24088
|
-
metadata: model
|
|
24089
|
-
metrics:
|
|
24440
|
+
metadata: { ...model && { model }, provider: "anthropic" },
|
|
24441
|
+
...Object.keys(metrics).length > 0 ? { metrics } : {},
|
|
24090
24442
|
output: outputs
|
|
24091
24443
|
});
|
|
24092
24444
|
const spanExport = await span.export();
|
|
@@ -24176,11 +24528,17 @@ function prepareLocalToolHandlersInMcpServers(mcpServers) {
|
|
|
24176
24528
|
}
|
|
24177
24529
|
return { hasLocalToolHandlers, localToolHookNames };
|
|
24178
24530
|
}
|
|
24179
|
-
function createToolTracingHooks(resolveParentSpan, activeToolSpans, mcpServers, localToolHookNames, skipLocalToolHooks, subAgentDetailsByToolUseId, subAgentSpans, endedSubAgentSpans) {
|
|
24531
|
+
function createToolTracingHooks(resolveParentSpan, taskIdToToolUseId, toolUseToParent, activeToolSpans, mcpServers, localToolHookNames, skipLocalToolHooks, subAgentDetailsByToolUseId, subAgentSpans, endedSubAgentSpans) {
|
|
24180
24532
|
const preToolUse = async (input, toolUseID) => {
|
|
24181
24533
|
if (input.hook_event_name !== "PreToolUse" || !toolUseID) {
|
|
24182
24534
|
return {};
|
|
24183
24535
|
}
|
|
24536
|
+
if (!toolUseToParent.has(toolUseID) && input.agent_id) {
|
|
24537
|
+
const parentToolUseId = taskIdToToolUseId.get(input.agent_id);
|
|
24538
|
+
if (parentToolUseId) {
|
|
24539
|
+
toolUseToParent.set(toolUseID, parentToolUseId);
|
|
24540
|
+
}
|
|
24541
|
+
}
|
|
24184
24542
|
if (skipLocalToolHooks && (isLocalToolUse(input.tool_name, mcpServers) || localToolHookNames.has(input.tool_name))) {
|
|
24185
24543
|
return {};
|
|
24186
24544
|
}
|
|
@@ -24376,9 +24734,6 @@ function createToolTracingHooks(resolveParentSpan, activeToolSpans, mcpServers,
|
|
|
24376
24734
|
}
|
|
24377
24735
|
const metadata = {
|
|
24378
24736
|
...subAgentDetailsToMetadata(details),
|
|
24379
|
-
...input.agent_transcript_path && {
|
|
24380
|
-
"claude_agent_sdk.agent_transcript_path": input.agent_transcript_path
|
|
24381
|
-
},
|
|
24382
24737
|
"claude_agent_sdk.stop_hook_active": input.stop_hook_active
|
|
24383
24738
|
};
|
|
24384
24739
|
try {
|
|
@@ -24400,7 +24755,7 @@ function createToolTracingHooks(resolveParentSpan, activeToolSpans, mcpServers,
|
|
|
24400
24755
|
subagentStop
|
|
24401
24756
|
};
|
|
24402
24757
|
}
|
|
24403
|
-
function injectTracingHooks(options, resolveParentSpan, activeToolSpans, localToolHookNames, skipLocalToolHooks, subAgentDetailsByToolUseId, subAgentSpans, endedSubAgentSpans) {
|
|
24758
|
+
function injectTracingHooks(options, resolveParentSpan, taskIdToToolUseId, toolUseToParent, activeToolSpans, localToolHookNames, skipLocalToolHooks, subAgentDetailsByToolUseId, subAgentSpans, endedSubAgentSpans) {
|
|
24404
24759
|
const {
|
|
24405
24760
|
preToolUse,
|
|
24406
24761
|
postToolUse,
|
|
@@ -24409,6 +24764,8 @@ function injectTracingHooks(options, resolveParentSpan, activeToolSpans, localTo
|
|
|
24409
24764
|
subagentStop
|
|
24410
24765
|
} = createToolTracingHooks(
|
|
24411
24766
|
resolveParentSpan,
|
|
24767
|
+
taskIdToToolUseId,
|
|
24768
|
+
toolUseToParent,
|
|
24412
24769
|
activeToolSpans,
|
|
24413
24770
|
options.mcpServers,
|
|
24414
24771
|
localToolHookNames,
|
|
@@ -24490,6 +24847,13 @@ async function finalizeCurrentMessageGroup(state) {
|
|
|
24490
24847
|
}
|
|
24491
24848
|
}
|
|
24492
24849
|
const existingLlmSpan = state.activeLlmSpansByParentToolUse.get(parentKey);
|
|
24850
|
+
const lastMessage = state.currentMessages[state.currentMessages.length - 1];
|
|
24851
|
+
const messageId = lastMessage?.message?.id;
|
|
24852
|
+
const usage = state.options.includePartialMessages ? mergeUsage(
|
|
24853
|
+
copyUsage(lastMessage?.message?.usage),
|
|
24854
|
+
messageId ? state.usageByMessageId.get(messageId) : void 0
|
|
24855
|
+
) : void 0;
|
|
24856
|
+
const hasFinalOutputUsage = messageId !== void 0 && state.finalOutputUsageMessageIds.has(messageId);
|
|
24493
24857
|
const llmSpanResult = await createLLMSpanForMessages(
|
|
24494
24858
|
state.currentMessages,
|
|
24495
24859
|
promptMessages,
|
|
@@ -24497,6 +24861,8 @@ async function finalizeCurrentMessageGroup(state) {
|
|
|
24497
24861
|
state.options,
|
|
24498
24862
|
state.currentMessageStartTime,
|
|
24499
24863
|
parentSpan,
|
|
24864
|
+
usage,
|
|
24865
|
+
hasFinalOutputUsage,
|
|
24500
24866
|
existingLlmSpan
|
|
24501
24867
|
);
|
|
24502
24868
|
if (llmSpanResult) {
|
|
@@ -24514,9 +24880,17 @@ async function finalizeCurrentMessageGroup(state) {
|
|
|
24514
24880
|
}
|
|
24515
24881
|
}
|
|
24516
24882
|
state.activeLlmSpansByParentToolUse.delete(parentKey);
|
|
24517
|
-
|
|
24518
|
-
|
|
24519
|
-
state.
|
|
24883
|
+
if (messageId) {
|
|
24884
|
+
state.usageByMessageId.delete(messageId);
|
|
24885
|
+
state.finalOutputUsageMessageIds.delete(messageId);
|
|
24886
|
+
for (const [
|
|
24887
|
+
parent,
|
|
24888
|
+
activeMessageId
|
|
24889
|
+
] of state.activePartialMessageIdByParentKey) {
|
|
24890
|
+
if (activeMessageId === messageId) {
|
|
24891
|
+
state.activePartialMessageIdByParentKey.delete(parent);
|
|
24892
|
+
}
|
|
24893
|
+
}
|
|
24520
24894
|
}
|
|
24521
24895
|
state.currentMessages.length = 0;
|
|
24522
24896
|
}
|
|
@@ -24607,6 +24981,10 @@ async function ensureActiveLlmSpanForParentToolUse(rootSpan, activeLlmSpansByPar
|
|
|
24607
24981
|
);
|
|
24608
24982
|
llmParentSpan = await subAgentSpan.export();
|
|
24609
24983
|
}
|
|
24984
|
+
const racedLlmSpan = activeLlmSpansByParentToolUse.get(parentKey);
|
|
24985
|
+
if (racedLlmSpan) {
|
|
24986
|
+
return racedLlmSpan;
|
|
24987
|
+
}
|
|
24610
24988
|
const llmSpan = startSpan(
|
|
24611
24989
|
withSpanInstrumentationName(
|
|
24612
24990
|
{
|
|
@@ -24726,7 +25104,49 @@ async function maybeHandleTaskLifecycleMessage(state, message) {
|
|
|
24726
25104
|
}
|
|
24727
25105
|
return true;
|
|
24728
25106
|
}
|
|
25107
|
+
function handlePartialUsageMessage(state, message) {
|
|
25108
|
+
if (message.type !== "stream_event") {
|
|
25109
|
+
return false;
|
|
25110
|
+
}
|
|
25111
|
+
const event = message.event;
|
|
25112
|
+
if (!event || typeof event !== "object") {
|
|
25113
|
+
return true;
|
|
25114
|
+
}
|
|
25115
|
+
const parentKey = llmParentKey(message.parent_tool_use_id ?? null);
|
|
25116
|
+
if (event.type === "message_start") {
|
|
25117
|
+
const messageId2 = event.message?.id;
|
|
25118
|
+
const usage = copyUsage(event.message?.usage);
|
|
25119
|
+
if (messageId2) {
|
|
25120
|
+
state.activePartialMessageIdByParentKey.set(parentKey, messageId2);
|
|
25121
|
+
if (usage) {
|
|
25122
|
+
state.usageByMessageId.set(messageId2, usage);
|
|
25123
|
+
}
|
|
25124
|
+
}
|
|
25125
|
+
return true;
|
|
25126
|
+
}
|
|
25127
|
+
const messageId = state.activePartialMessageIdByParentKey.get(parentKey);
|
|
25128
|
+
if (!messageId) {
|
|
25129
|
+
return true;
|
|
25130
|
+
}
|
|
25131
|
+
if (event.type === "message_delta") {
|
|
25132
|
+
const update = copyUsage(event.usage);
|
|
25133
|
+
if (update) {
|
|
25134
|
+
const usage = state.usageByMessageId.get(messageId) ?? {};
|
|
25135
|
+
Object.assign(usage, update);
|
|
25136
|
+
state.usageByMessageId.set(messageId, usage);
|
|
25137
|
+
if (update.output_tokens !== void 0) {
|
|
25138
|
+
state.finalOutputUsageMessageIds.add(messageId);
|
|
25139
|
+
}
|
|
25140
|
+
}
|
|
25141
|
+
} else if (event.type === "message_stop") {
|
|
25142
|
+
state.activePartialMessageIdByParentKey.delete(parentKey);
|
|
25143
|
+
}
|
|
25144
|
+
return true;
|
|
25145
|
+
}
|
|
24729
25146
|
async function handleStreamMessage(state, message) {
|
|
25147
|
+
if (handlePartialUsageMessage(state, message)) {
|
|
25148
|
+
return;
|
|
25149
|
+
}
|
|
24730
25150
|
maybeTrackToolUseContext(state, message);
|
|
24731
25151
|
if (await maybeHandleTaskLifecycleMessage(state, message)) {
|
|
24732
25152
|
return;
|
|
@@ -24773,36 +25193,9 @@ async function handleStreamMessage(state, message) {
|
|
|
24773
25193
|
);
|
|
24774
25194
|
state.currentMessages.push(message);
|
|
24775
25195
|
}
|
|
24776
|
-
if (message.type !== "result"
|
|
25196
|
+
if (message.type !== "result") {
|
|
24777
25197
|
return;
|
|
24778
25198
|
}
|
|
24779
|
-
const finalUsageMetrics = extractUsageFromMessage(message);
|
|
24780
|
-
if (state.currentMessages.length > 0 && finalUsageMetrics.completion_tokens !== void 0) {
|
|
24781
|
-
const lastMessage = state.currentMessages[state.currentMessages.length - 1];
|
|
24782
|
-
if (lastMessage?.message?.usage) {
|
|
24783
|
-
const adjustedTokens = finalUsageMetrics.completion_tokens - state.accumulatedOutputTokens;
|
|
24784
|
-
if (adjustedTokens >= 0) {
|
|
24785
|
-
lastMessage.message.usage.output_tokens = adjustedTokens;
|
|
24786
|
-
}
|
|
24787
|
-
const resultUsage = message.usage;
|
|
24788
|
-
if (resultUsage && typeof resultUsage === "object") {
|
|
24789
|
-
const cacheReadTokens = getNumberProperty(
|
|
24790
|
-
resultUsage,
|
|
24791
|
-
"cache_read_input_tokens"
|
|
24792
|
-
);
|
|
24793
|
-
if (cacheReadTokens !== void 0) {
|
|
24794
|
-
lastMessage.message.usage.cache_read_input_tokens = cacheReadTokens;
|
|
24795
|
-
}
|
|
24796
|
-
const cacheCreationTokens = getNumberProperty(
|
|
24797
|
-
resultUsage,
|
|
24798
|
-
"cache_creation_input_tokens"
|
|
24799
|
-
);
|
|
24800
|
-
if (cacheCreationTokens !== void 0) {
|
|
24801
|
-
lastMessage.message.usage.cache_creation_input_tokens = cacheCreationTokens;
|
|
24802
|
-
}
|
|
24803
|
-
}
|
|
24804
|
-
}
|
|
24805
|
-
}
|
|
24806
25199
|
const metadata = {};
|
|
24807
25200
|
if (message.num_turns !== void 0) {
|
|
24808
25201
|
metadata.num_turns = message.num_turns;
|
|
@@ -24810,8 +25203,12 @@ async function handleStreamMessage(state, message) {
|
|
|
24810
25203
|
if (message.session_id !== void 0) {
|
|
24811
25204
|
metadata.session_id = message.session_id;
|
|
24812
25205
|
}
|
|
24813
|
-
|
|
24814
|
-
|
|
25206
|
+
const metrics = state.options.includePartialMessages ? {} : extractUsage(copyUsage(message.usage), true);
|
|
25207
|
+
if (Object.keys(metadata).length > 0 || Object.keys(metrics).length > 0) {
|
|
25208
|
+
state.span.log({
|
|
25209
|
+
...Object.keys(metadata).length > 0 ? { metadata } : {},
|
|
25210
|
+
...Object.keys(metrics).length > 0 ? { metrics } : {}
|
|
25211
|
+
});
|
|
24815
25212
|
}
|
|
24816
25213
|
}
|
|
24817
25214
|
async function finalizeQuerySpan(state) {
|
|
@@ -24835,6 +25232,9 @@ async function finalizeQuerySpan(state) {
|
|
|
24835
25232
|
llmSpan.end();
|
|
24836
25233
|
}
|
|
24837
25234
|
state.activeLlmSpansByParentToolUse.clear();
|
|
25235
|
+
state.activePartialMessageIdByParentKey.clear();
|
|
25236
|
+
state.finalOutputUsageMessageIds.clear();
|
|
25237
|
+
state.usageByMessageId.clear();
|
|
24838
25238
|
for (const toolSpan of state.activeToolSpans.values()) {
|
|
24839
25239
|
toolSpan.end();
|
|
24840
25240
|
}
|
|
@@ -24959,6 +25359,8 @@ var ClaudeAgentSDKPlugin = class extends BasePlugin {
|
|
|
24959
25359
|
const optionsWithHooks = injectTracingHooks(
|
|
24960
25360
|
options,
|
|
24961
25361
|
resolveToolUseParentSpan,
|
|
25362
|
+
taskIdToToolUseId,
|
|
25363
|
+
toolUseToParent,
|
|
24962
25364
|
activeToolSpans,
|
|
24963
25365
|
localToolHookNames,
|
|
24964
25366
|
skipLocalToolHooks,
|
|
@@ -24969,8 +25371,8 @@ var ClaudeAgentSDKPlugin = class extends BasePlugin {
|
|
|
24969
25371
|
params.options = optionsWithHooks;
|
|
24970
25372
|
event.arguments[0] = params;
|
|
24971
25373
|
spans.set(event, {
|
|
24972
|
-
accumulatedOutputTokens: 0,
|
|
24973
25374
|
activeLlmSpansByParentToolUse,
|
|
25375
|
+
activePartialMessageIdByParentKey: /* @__PURE__ */ new Map(),
|
|
24974
25376
|
activeToolSpans,
|
|
24975
25377
|
conversationHistoryByParentKey,
|
|
24976
25378
|
capturedPromptMessages,
|
|
@@ -24978,6 +25380,7 @@ var ClaudeAgentSDKPlugin = class extends BasePlugin {
|
|
|
24978
25380
|
currentMessageStartTime: startTime,
|
|
24979
25381
|
currentMessages: [],
|
|
24980
25382
|
endedSubAgentSpans,
|
|
25383
|
+
finalOutputUsageMessageIds: /* @__PURE__ */ new Set(),
|
|
24981
25384
|
finalResults: [],
|
|
24982
25385
|
options: optionsWithHooks,
|
|
24983
25386
|
originalPrompt,
|
|
@@ -24993,6 +25396,7 @@ var ClaudeAgentSDKPlugin = class extends BasePlugin {
|
|
|
24993
25396
|
latestLlmParentBySubAgentToolUse,
|
|
24994
25397
|
latestRootLlmParentRef,
|
|
24995
25398
|
toolUseToParent,
|
|
25399
|
+
usageByMessageId: /* @__PURE__ */ new Map(),
|
|
24996
25400
|
localToolContext
|
|
24997
25401
|
});
|
|
24998
25402
|
},
|
|
@@ -27467,9 +27871,9 @@ function extractEmbedPromptTokenCount(response) {
|
|
|
27467
27871
|
let sawAny = false;
|
|
27468
27872
|
for (const embedding of embeddings) {
|
|
27469
27873
|
const embeddingStats = tryToDict(tryToDict(embedding)?.statistics);
|
|
27470
|
-
const
|
|
27471
|
-
if (typeof
|
|
27472
|
-
total +=
|
|
27874
|
+
const tokenCount2 = embeddingStats?.tokenCount;
|
|
27875
|
+
if (typeof tokenCount2 === "number" && Number.isFinite(tokenCount2)) {
|
|
27876
|
+
total += tokenCount2;
|
|
27473
27877
|
sawAny = true;
|
|
27474
27878
|
}
|
|
27475
27879
|
}
|
|
@@ -34364,7 +34768,7 @@ function getStringProperty2(obj, key) {
|
|
|
34364
34768
|
return typeof value === "string" ? value : void 0;
|
|
34365
34769
|
}
|
|
34366
34770
|
function extractMetricsFromUsage(usage) {
|
|
34367
|
-
const
|
|
34771
|
+
const rawMetrics = {
|
|
34368
34772
|
prompt_tokens: usage.inputTokens,
|
|
34369
34773
|
completion_tokens: usage.outputTokens,
|
|
34370
34774
|
...extractAnthropicCacheTokens(
|
|
@@ -34373,10 +34777,10 @@ function extractMetricsFromUsage(usage) {
|
|
|
34373
34777
|
)
|
|
34374
34778
|
};
|
|
34375
34779
|
if (usage.reasoningTokens !== void 0) {
|
|
34376
|
-
|
|
34377
|
-
|
|
34780
|
+
rawMetrics.completion_reasoning_tokens = usage.reasoningTokens;
|
|
34781
|
+
rawMetrics.reasoning_tokens = usage.reasoningTokens;
|
|
34378
34782
|
}
|
|
34379
|
-
|
|
34783
|
+
const metrics = finalizeAnthropicTokens(rawMetrics);
|
|
34380
34784
|
const metadata = {
|
|
34381
34785
|
model: usage.model
|
|
34382
34786
|
};
|
|
@@ -35315,17 +35719,20 @@ var FlueObserveBridge = class {
|
|
|
35315
35719
|
return;
|
|
35316
35720
|
}
|
|
35317
35721
|
const metadata = {
|
|
35722
|
+
...event.runId ? this.runsById.get(event.runId)?.metadata : {},
|
|
35318
35723
|
...extractEventMetadata(event),
|
|
35319
35724
|
"flue.operation": event.operationKind,
|
|
35320
35725
|
provider: "flue"
|
|
35321
35726
|
};
|
|
35322
35727
|
const parent = this.parentSpanForEvent(event);
|
|
35323
|
-
const
|
|
35728
|
+
const args = {
|
|
35324
35729
|
name: `flue.${event.operationKind}`,
|
|
35325
35730
|
spanAttributes: { type: "task" /* TASK */ },
|
|
35326
35731
|
startTime: eventTime(event.timestamp),
|
|
35327
35732
|
event: { metadata }
|
|
35328
|
-
}
|
|
35733
|
+
};
|
|
35734
|
+
const runSpan = event.runId ? this.runsById.get(event.runId)?.span : void 0;
|
|
35735
|
+
const span = event.operationKind === "prompt" && (!parent || parent === runSpan) ? startFlueRootSpan(args) : startFlueSpan(parent, args);
|
|
35329
35736
|
this.operationsById.set(event.operationId, { metadata, span });
|
|
35330
35737
|
}
|
|
35331
35738
|
handleOperation(event) {
|
|
@@ -35340,6 +35747,11 @@ var FlueObserveBridge = class {
|
|
|
35340
35747
|
...event.isError !== void 0 ? { "flue.is_error": event.isError } : {},
|
|
35341
35748
|
...event.usage ? { "flue.usage": event.usage } : {}
|
|
35342
35749
|
};
|
|
35750
|
+
const input = flueOperationInput(event);
|
|
35751
|
+
if (!state.loggedInput && input !== void 0) {
|
|
35752
|
+
safeLog3(state.span, { input });
|
|
35753
|
+
state.loggedInput = true;
|
|
35754
|
+
}
|
|
35343
35755
|
this.finishPendingChildrenForOperation(event, output);
|
|
35344
35756
|
safeLog3(state.span, {
|
|
35345
35757
|
...event.isError ? { error: toLoggedError(event.errorInfo ?? event.error) } : {},
|
|
@@ -35356,6 +35768,8 @@ var FlueObserveBridge = class {
|
|
|
35356
35768
|
return;
|
|
35357
35769
|
}
|
|
35358
35770
|
const input = flueTurnRequestInput(event);
|
|
35771
|
+
const operation = event.operationId ? this.operationsById.get(event.operationId) : void 0;
|
|
35772
|
+
const turnInput = prepareFlueTurnInput(event, input, operation);
|
|
35359
35773
|
const model = flueTurnRequestModel(event);
|
|
35360
35774
|
const provider = flueTurnRequestProvider(event);
|
|
35361
35775
|
const api = flueTurnRequestApi(event);
|
|
@@ -35368,8 +35782,7 @@ var FlueObserveBridge = class {
|
|
|
35368
35782
|
...provider ? { "flue.provider": provider } : {},
|
|
35369
35783
|
...event.purpose ? { "flue.turn_purpose": event.purpose } : {},
|
|
35370
35784
|
...reasoning ? { reasoning } : {},
|
|
35371
|
-
...
|
|
35372
|
-
...input?.tools ? { tools: input.tools } : {}
|
|
35785
|
+
...turnInput.metadata
|
|
35373
35786
|
};
|
|
35374
35787
|
const parent = this.parentSpanForTurn(event);
|
|
35375
35788
|
const span = startFlueSpan(parent, {
|
|
@@ -35377,11 +35790,14 @@ var FlueObserveBridge = class {
|
|
|
35377
35790
|
spanAttributes: { type: "llm" /* LLM */ },
|
|
35378
35791
|
startTime: eventTime(event.timestamp),
|
|
35379
35792
|
event: {
|
|
35380
|
-
input:
|
|
35793
|
+
input: turnInput.messages,
|
|
35381
35794
|
metadata
|
|
35382
35795
|
}
|
|
35383
35796
|
});
|
|
35384
|
-
this.logOperationInput(
|
|
35797
|
+
this.logOperationInput(
|
|
35798
|
+
event.operationId,
|
|
35799
|
+
latestUserMessageInput(input?.messages)
|
|
35800
|
+
);
|
|
35385
35801
|
this.turnsByKey.set(key, { metadata, span });
|
|
35386
35802
|
}
|
|
35387
35803
|
handleTurn(event) {
|
|
@@ -35628,16 +36044,20 @@ var FlueObserveBridge = class {
|
|
|
35628
36044
|
}
|
|
35629
36045
|
startSyntheticOperation(event) {
|
|
35630
36046
|
const metadata = {
|
|
36047
|
+
...event.runId ? this.runsById.get(event.runId)?.metadata : {},
|
|
35631
36048
|
...extractEventMetadata(event),
|
|
35632
36049
|
"flue.operation": event.operationKind,
|
|
35633
36050
|
provider: "flue"
|
|
35634
36051
|
};
|
|
35635
|
-
const
|
|
36052
|
+
const args = {
|
|
35636
36053
|
name: `flue.${event.operationKind}`,
|
|
35637
36054
|
spanAttributes: { type: "task" /* TASK */ },
|
|
35638
36055
|
startTime: eventTime(event.timestamp),
|
|
35639
36056
|
event: { metadata }
|
|
35640
|
-
}
|
|
36057
|
+
};
|
|
36058
|
+
const parent = this.parentSpanForEvent(event);
|
|
36059
|
+
const runSpan = event.runId ? this.runsById.get(event.runId)?.span : void 0;
|
|
36060
|
+
const span = event.operationKind === "prompt" && (!parent || parent === runSpan) ? startFlueRootSpan(args) : startFlueSpan(parent, args);
|
|
35641
36061
|
return { metadata, span };
|
|
35642
36062
|
}
|
|
35643
36063
|
startSyntheticTurn(event) {
|
|
@@ -35816,6 +36236,71 @@ function flueRunInput(event) {
|
|
|
35816
36236
|
function flueTurnRequestInput(event) {
|
|
35817
36237
|
return event.request?.input ?? event.input;
|
|
35818
36238
|
}
|
|
36239
|
+
function prepareFlueTurnInput(event, input, operation) {
|
|
36240
|
+
const messages = input?.messages;
|
|
36241
|
+
const tracksUserTurn = event.purpose === "agent" && operation?.metadata["flue.operation"] === "prompt" && Array.isArray(messages);
|
|
36242
|
+
if (!tracksUserTurn) {
|
|
36243
|
+
return {
|
|
36244
|
+
messages,
|
|
36245
|
+
metadata: {
|
|
36246
|
+
...input?.systemPrompt ? { "flue.system_prompt": input.systemPrompt } : {},
|
|
36247
|
+
...input?.tools ? { tools: input.tools } : {}
|
|
36248
|
+
}
|
|
36249
|
+
};
|
|
36250
|
+
}
|
|
36251
|
+
const previous = operation.turnInputState;
|
|
36252
|
+
const previousMessageCount = previous?.messageCount ?? 0;
|
|
36253
|
+
const boundaryFingerprint = messages.length > 0 ? fingerprintJsonValue(messages[messages.length - 1]) : void 0;
|
|
36254
|
+
const continuesPreviousInput = previous !== void 0 && messages.length >= previousMessageCount && (previousMessageCount === 0 || previous.boundaryFingerprint !== void 0 && previous.boundaryFingerprint === fingerprintJsonValue(messages[previousMessageCount - 1]));
|
|
36255
|
+
const inputMode = previous === void 0 ? "full" : continuesPreviousInput ? "delta" : "reset";
|
|
36256
|
+
const systemPromptFingerprint = fingerprintJsonValue(input?.systemPrompt);
|
|
36257
|
+
const toolsFingerprint = fingerprintJsonValue(input?.tools);
|
|
36258
|
+
operation.turnInputState = {
|
|
36259
|
+
...boundaryFingerprint !== void 0 ? { boundaryFingerprint } : {},
|
|
36260
|
+
messageCount: messages.length,
|
|
36261
|
+
...systemPromptFingerprint !== void 0 ? { systemPromptFingerprint } : {},
|
|
36262
|
+
...toolsFingerprint !== void 0 ? { toolsFingerprint } : {}
|
|
36263
|
+
};
|
|
36264
|
+
return {
|
|
36265
|
+
messages: continuesPreviousInput ? messages.slice(previousMessageCount) : messages,
|
|
36266
|
+
metadata: {
|
|
36267
|
+
"flue.input_mode": inputMode,
|
|
36268
|
+
...continuesPreviousInput ? { "flue.input_message_offset": previousMessageCount } : {},
|
|
36269
|
+
...input?.systemPrompt && (!continuesPreviousInput || systemPromptFingerprint !== previous?.systemPromptFingerprint) ? { "flue.system_prompt": input.systemPrompt } : {},
|
|
36270
|
+
...input?.tools && (!continuesPreviousInput || toolsFingerprint !== previous?.toolsFingerprint) ? { tools: input.tools } : {}
|
|
36271
|
+
}
|
|
36272
|
+
};
|
|
36273
|
+
}
|
|
36274
|
+
function fingerprintJsonValue(value) {
|
|
36275
|
+
try {
|
|
36276
|
+
const serialized = JSON.stringify(value);
|
|
36277
|
+
if (serialized === void 0) {
|
|
36278
|
+
return void 0;
|
|
36279
|
+
}
|
|
36280
|
+
let hash = 2166136261;
|
|
36281
|
+
for (let i = 0; i < serialized.length; i++) {
|
|
36282
|
+
hash = Math.imul(hash ^ serialized.charCodeAt(i), 16777619);
|
|
36283
|
+
}
|
|
36284
|
+
return `${serialized.length}:${hash >>> 0}`;
|
|
36285
|
+
} catch {
|
|
36286
|
+
return void 0;
|
|
36287
|
+
}
|
|
36288
|
+
}
|
|
36289
|
+
function latestUserMessageInput(messages) {
|
|
36290
|
+
if (!messages) {
|
|
36291
|
+
return void 0;
|
|
36292
|
+
}
|
|
36293
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
36294
|
+
const message = messages[i];
|
|
36295
|
+
if (isObjectLike(message) && Reflect.get(message, "role") === "user") {
|
|
36296
|
+
return [message];
|
|
36297
|
+
}
|
|
36298
|
+
}
|
|
36299
|
+
return void 0;
|
|
36300
|
+
}
|
|
36301
|
+
function flueOperationInput(event) {
|
|
36302
|
+
return typeof event.agentInput?.text === "string" ? [{ content: event.agentInput.text, role: "user" }] : void 0;
|
|
36303
|
+
}
|
|
35819
36304
|
function flueTurnRequestModel(event) {
|
|
35820
36305
|
return event.request?.requestedModel ?? event.request?.model ?? event.model;
|
|
35821
36306
|
}
|
|
@@ -35973,6 +36458,21 @@ function startFlueSpan(parent, args) {
|
|
|
35973
36458
|
withSpanInstrumentationName(args, INSTRUMENTATION_NAMES.FLUE)
|
|
35974
36459
|
);
|
|
35975
36460
|
}
|
|
36461
|
+
function startFlueRootSpan(args) {
|
|
36462
|
+
const state = _internalGetGlobalState();
|
|
36463
|
+
const spanId = state.idGenerator.getSpanId();
|
|
36464
|
+
const rootSpanId = state.idGenerator.shareRootSpanId() ? spanId : state.idGenerator.getTraceId();
|
|
36465
|
+
return withCurrent(
|
|
36466
|
+
NOOP_SPAN,
|
|
36467
|
+
() => startSpan({
|
|
36468
|
+
...withSpanInstrumentationName(args, INSTRUMENTATION_NAMES.FLUE),
|
|
36469
|
+
parentSpanIds: { parentSpanIds: [], rootSpanId },
|
|
36470
|
+
spanId,
|
|
36471
|
+
state
|
|
36472
|
+
}),
|
|
36473
|
+
state
|
|
36474
|
+
);
|
|
36475
|
+
}
|
|
35976
36476
|
function runWithCurrentSpanStore(span, next) {
|
|
35977
36477
|
const state = _internalGetGlobalState();
|
|
35978
36478
|
const contextManager = state?.contextManager;
|
|
@@ -37872,8 +38372,7 @@ function startAgentStream(event, activeChildParents) {
|
|
|
37872
38372
|
...extractAgentMetadata2(agent),
|
|
37873
38373
|
...extractModelMetadata3(model),
|
|
37874
38374
|
"strands.operation": "Agent.stream",
|
|
37875
|
-
provider: extractProvider(model)
|
|
37876
|
-
...event.moduleVersion ? { "strands_agent_sdk.version": event.moduleVersion } : {}
|
|
38375
|
+
provider: extractProvider(model)
|
|
37877
38376
|
};
|
|
37878
38377
|
const parentSpan = agent ? getOnlyChildParent(activeChildParents, agent) : void 0;
|
|
37879
38378
|
const attachmentCache = createStrandsAttachmentCache();
|
|
@@ -37923,8 +38422,7 @@ function startMultiAgentStream(event, operation, activeChildParents) {
|
|
|
37923
38422
|
const metadata = {
|
|
37924
38423
|
"strands.operation": operation,
|
|
37925
38424
|
provider: "strands",
|
|
37926
|
-
...orchestrator?.id ? { "strands.orchestrator.id": orchestrator.id } : {}
|
|
37927
|
-
...event.moduleVersion ? { "strands_agent_sdk.version": event.moduleVersion } : {}
|
|
38425
|
+
...orchestrator?.id ? { "strands.orchestrator.id": orchestrator.id } : {}
|
|
37928
38426
|
};
|
|
37929
38427
|
const parentSpan = orchestrator ? getOnlyChildParent(activeChildParents, orchestrator) : void 0;
|
|
37930
38428
|
const input = processStrandsInputAttachments(event.arguments[0]);
|
|
@@ -38710,6 +39208,320 @@ function logInstrumentationError5(context2, error2) {
|
|
|
38710
39208
|
debugLogger.debug(`${context2}:`, error2);
|
|
38711
39209
|
}
|
|
38712
39210
|
|
|
39211
|
+
// src/instrumentation/plugins/voyageai-channels.ts
|
|
39212
|
+
var voyageAIChannels = defineChannels(
|
|
39213
|
+
"voyageai",
|
|
39214
|
+
{
|
|
39215
|
+
embed: channel({
|
|
39216
|
+
channelName: "embed",
|
|
39217
|
+
kind: "async"
|
|
39218
|
+
}),
|
|
39219
|
+
multimodalEmbed: channel({
|
|
39220
|
+
channelName: "multimodalEmbed",
|
|
39221
|
+
kind: "async"
|
|
39222
|
+
}),
|
|
39223
|
+
rerank: channel({
|
|
39224
|
+
channelName: "rerank",
|
|
39225
|
+
kind: "async"
|
|
39226
|
+
}),
|
|
39227
|
+
contextualizedEmbed: channel({
|
|
39228
|
+
channelName: "contextualizedEmbed",
|
|
39229
|
+
kind: "async"
|
|
39230
|
+
})
|
|
39231
|
+
},
|
|
39232
|
+
{ instrumentationName: INSTRUMENTATION_NAMES.VOYAGEAI }
|
|
39233
|
+
);
|
|
39234
|
+
|
|
39235
|
+
// src/instrumentation/plugins/voyageai-plugin.ts
|
|
39236
|
+
var RERANK_METADATA_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
39237
|
+
"model",
|
|
39238
|
+
"returnDocuments",
|
|
39239
|
+
"topK",
|
|
39240
|
+
"truncation"
|
|
39241
|
+
]);
|
|
39242
|
+
var VoyageAIPlugin = class extends BasePlugin {
|
|
39243
|
+
onEnable() {
|
|
39244
|
+
this.unsubscribers.push(
|
|
39245
|
+
interceptVoyageAICall(
|
|
39246
|
+
voyageAIChannels.embed,
|
|
39247
|
+
"voyageai.embed",
|
|
39248
|
+
extractTextEmbeddingInput,
|
|
39249
|
+
summarizeEmbeddingOutput,
|
|
39250
|
+
extractEmbeddingUsageMetrics
|
|
39251
|
+
),
|
|
39252
|
+
interceptVoyageAICall(
|
|
39253
|
+
voyageAIChannels.multimodalEmbed,
|
|
39254
|
+
"voyageai.multimodalEmbed",
|
|
39255
|
+
extractMultimodalEmbeddingInput,
|
|
39256
|
+
summarizeEmbeddingOutput,
|
|
39257
|
+
extractEmbeddingUsageMetrics
|
|
39258
|
+
),
|
|
39259
|
+
interceptVoyageAICall(
|
|
39260
|
+
voyageAIChannels.rerank,
|
|
39261
|
+
"voyageai.rerank",
|
|
39262
|
+
extractRerankInput,
|
|
39263
|
+
summarizeRerankOutput
|
|
39264
|
+
),
|
|
39265
|
+
interceptVoyageAICall(
|
|
39266
|
+
voyageAIChannels.contextualizedEmbed,
|
|
39267
|
+
"voyageai.contextualizedEmbed",
|
|
39268
|
+
extractContextualizedEmbeddingInput,
|
|
39269
|
+
summarizeContextualizedEmbeddingOutput,
|
|
39270
|
+
extractEmbeddingUsageMetrics
|
|
39271
|
+
)
|
|
39272
|
+
);
|
|
39273
|
+
}
|
|
39274
|
+
onDisable() {
|
|
39275
|
+
this.unsubscribers = unsubscribeAll(this.unsubscribers);
|
|
39276
|
+
}
|
|
39277
|
+
};
|
|
39278
|
+
function interceptVoyageAICall(channel2, name, extractInput2, extractOutput2, extractMetrics2 = extractUsageMetrics3) {
|
|
39279
|
+
return channel2.intercept((target, thisArg, args) => {
|
|
39280
|
+
const invokeTarget = () => Reflect.apply(target, thisArg, args);
|
|
39281
|
+
if (isAutoInstrumentationSuppressed()) {
|
|
39282
|
+
return invokeTarget();
|
|
39283
|
+
}
|
|
39284
|
+
let span;
|
|
39285
|
+
try {
|
|
39286
|
+
const { input, metadata } = extractInput2(args);
|
|
39287
|
+
span = startSpan(
|
|
39288
|
+
withSpanInstrumentationName(
|
|
39289
|
+
{
|
|
39290
|
+
event: { input, metadata },
|
|
39291
|
+
name,
|
|
39292
|
+
spanAttributes: { type: "llm" /* LLM */ }
|
|
39293
|
+
},
|
|
39294
|
+
INSTRUMENTATION_NAMES.VOYAGEAI
|
|
39295
|
+
)
|
|
39296
|
+
);
|
|
39297
|
+
} catch (error2) {
|
|
39298
|
+
debugLogger.error(`Error starting span for ${name}:`, error2);
|
|
39299
|
+
return invokeTarget();
|
|
39300
|
+
}
|
|
39301
|
+
let result;
|
|
39302
|
+
try {
|
|
39303
|
+
result = withCurrent(
|
|
39304
|
+
span,
|
|
39305
|
+
() => runWithAutoInstrumentationSuppressed(invokeTarget)
|
|
39306
|
+
);
|
|
39307
|
+
} catch (error2) {
|
|
39308
|
+
finishVoyageAISpan(span, name, () => span.log({ error: error2 }));
|
|
39309
|
+
throw error2;
|
|
39310
|
+
}
|
|
39311
|
+
void Promise.resolve(result).then(
|
|
39312
|
+
(value) => finishVoyageAISpan(span, name, () => {
|
|
39313
|
+
const metadata = extractResponseMetadata3(value);
|
|
39314
|
+
span.log({
|
|
39315
|
+
output: extractOutput2(value),
|
|
39316
|
+
...metadata ? { metadata } : {},
|
|
39317
|
+
metrics: extractMetrics2(value)
|
|
39318
|
+
});
|
|
39319
|
+
}),
|
|
39320
|
+
(error2) => finishVoyageAISpan(span, name, () => span.log({ error: error2 }))
|
|
39321
|
+
);
|
|
39322
|
+
return result;
|
|
39323
|
+
});
|
|
39324
|
+
}
|
|
39325
|
+
function finishVoyageAISpan(span, name, log2) {
|
|
39326
|
+
try {
|
|
39327
|
+
log2();
|
|
39328
|
+
} catch (error2) {
|
|
39329
|
+
debugLogger.error(`Error logging span for ${name}:`, error2);
|
|
39330
|
+
}
|
|
39331
|
+
try {
|
|
39332
|
+
span.end();
|
|
39333
|
+
} catch (error2) {
|
|
39334
|
+
debugLogger.error(`Error ending span for ${name}:`, error2);
|
|
39335
|
+
}
|
|
39336
|
+
}
|
|
39337
|
+
function getRequestArg2(args) {
|
|
39338
|
+
if (Array.isArray(args)) {
|
|
39339
|
+
return isObject(args[0]) ? args[0] : void 0;
|
|
39340
|
+
}
|
|
39341
|
+
if (!isObject(args)) {
|
|
39342
|
+
return void 0;
|
|
39343
|
+
}
|
|
39344
|
+
const firstArg = Reflect.get(args, "0");
|
|
39345
|
+
return isObject(firstArg) ? firstArg : void 0;
|
|
39346
|
+
}
|
|
39347
|
+
function pickMetadata(request, allowlist) {
|
|
39348
|
+
const metadata = {};
|
|
39349
|
+
if (request) {
|
|
39350
|
+
for (const key of allowlist) {
|
|
39351
|
+
if (!Object.hasOwn(request, key)) {
|
|
39352
|
+
continue;
|
|
39353
|
+
}
|
|
39354
|
+
const value = request[key];
|
|
39355
|
+
if (value !== void 0) {
|
|
39356
|
+
metadata[key] = value;
|
|
39357
|
+
}
|
|
39358
|
+
}
|
|
39359
|
+
}
|
|
39360
|
+
return {
|
|
39361
|
+
...metadata,
|
|
39362
|
+
provider: "voyage"
|
|
39363
|
+
};
|
|
39364
|
+
}
|
|
39365
|
+
function buildEmbeddingInput(inputs, request) {
|
|
39366
|
+
const outputDimensions = request?.outputDimension;
|
|
39367
|
+
return {
|
|
39368
|
+
inputs,
|
|
39369
|
+
...typeof outputDimensions === "number" && Number.isFinite(outputDimensions) ? { output_dimensions: outputDimensions } : {}
|
|
39370
|
+
};
|
|
39371
|
+
}
|
|
39372
|
+
function embeddingMetadata(request) {
|
|
39373
|
+
return {
|
|
39374
|
+
...typeof request?.model === "string" ? { model: request.model } : {},
|
|
39375
|
+
provider: "voyage"
|
|
39376
|
+
};
|
|
39377
|
+
}
|
|
39378
|
+
function extractTextEmbeddingInput(args) {
|
|
39379
|
+
const request = getRequestArg2(args);
|
|
39380
|
+
const rawInput = request?.input;
|
|
39381
|
+
const values = Array.isArray(rawInput) ? rawInput : [rawInput];
|
|
39382
|
+
return {
|
|
39383
|
+
input: buildEmbeddingInput(
|
|
39384
|
+
values.flatMap(
|
|
39385
|
+
(value) => typeof value === "string" ? [{ content: value }] : []
|
|
39386
|
+
),
|
|
39387
|
+
request
|
|
39388
|
+
),
|
|
39389
|
+
metadata: embeddingMetadata(request)
|
|
39390
|
+
};
|
|
39391
|
+
}
|
|
39392
|
+
function extractContextualizedEmbeddingInput(args) {
|
|
39393
|
+
const request = getRequestArg2(args);
|
|
39394
|
+
const rawInputs = request?.inputs;
|
|
39395
|
+
const values = Array.isArray(rawInputs) ? rawInputs.flatMap((value) => Array.isArray(value) ? value : [value]) : [];
|
|
39396
|
+
return {
|
|
39397
|
+
input: buildEmbeddingInput(
|
|
39398
|
+
values.flatMap(
|
|
39399
|
+
(value) => typeof value === "string" ? [{ content: value }] : []
|
|
39400
|
+
),
|
|
39401
|
+
request
|
|
39402
|
+
),
|
|
39403
|
+
metadata: embeddingMetadata(request)
|
|
39404
|
+
};
|
|
39405
|
+
}
|
|
39406
|
+
function extractMultimodalEmbeddingInput(args) {
|
|
39407
|
+
const request = getRequestArg2(args);
|
|
39408
|
+
const rawInputs = request?.inputs;
|
|
39409
|
+
const inputs = Array.isArray(rawInputs) ? rawInputs.map((rawInput) => {
|
|
39410
|
+
const rawContent = isObject(rawInput) ? rawInput.content : void 0;
|
|
39411
|
+
return {
|
|
39412
|
+
content: Array.isArray(rawContent) ? rawContent.flatMap(normalizeMultimodalContentPart) : []
|
|
39413
|
+
};
|
|
39414
|
+
}) : [];
|
|
39415
|
+
const input = buildEmbeddingInput(inputs, request);
|
|
39416
|
+
const processedInput = processInputAttachments(input);
|
|
39417
|
+
return {
|
|
39418
|
+
input: hasInlineEmbeddingMedia(processedInput) ? input : processedInput,
|
|
39419
|
+
metadata: embeddingMetadata(request)
|
|
39420
|
+
};
|
|
39421
|
+
}
|
|
39422
|
+
function normalizeMultimodalContentPart(part) {
|
|
39423
|
+
if (!isObject(part) || typeof part.type !== "string") {
|
|
39424
|
+
return [];
|
|
39425
|
+
}
|
|
39426
|
+
if (part.type === "text") {
|
|
39427
|
+
return typeof part.text === "string" ? [{ type: "text", text: part.text }] : [];
|
|
39428
|
+
}
|
|
39429
|
+
const camelCaseField = {
|
|
39430
|
+
image_base64: "imageBase64",
|
|
39431
|
+
image_url: "imageUrl",
|
|
39432
|
+
video_base64: "videoBase64",
|
|
39433
|
+
video_url: "videoUrl"
|
|
39434
|
+
};
|
|
39435
|
+
const field = camelCaseField[part.type];
|
|
39436
|
+
if (!field) {
|
|
39437
|
+
return [];
|
|
39438
|
+
}
|
|
39439
|
+
const data = typeof part[field] === "string" ? part[field] : part[part.type];
|
|
39440
|
+
if (typeof data !== "string") {
|
|
39441
|
+
return [];
|
|
39442
|
+
}
|
|
39443
|
+
return part.type.startsWith("image_") ? [{ type: "image_url", image_url: { url: data } }] : [{ type: "file", file: { file_data: data } }];
|
|
39444
|
+
}
|
|
39445
|
+
function hasInlineEmbeddingMedia(input) {
|
|
39446
|
+
return input.inputs.some(
|
|
39447
|
+
({ content }) => Array.isArray(content) ? content.some((part) => {
|
|
39448
|
+
const value = part.type === "image_url" ? part.image_url.url : part.type === "file" ? part.file.file_data : void 0;
|
|
39449
|
+
return typeof value === "string" && value.startsWith("data:");
|
|
39450
|
+
}) : false
|
|
39451
|
+
);
|
|
39452
|
+
}
|
|
39453
|
+
function extractRerankInput(args) {
|
|
39454
|
+
const request = getRequestArg2(args);
|
|
39455
|
+
const documents = request?.documents;
|
|
39456
|
+
return {
|
|
39457
|
+
input: {
|
|
39458
|
+
documents,
|
|
39459
|
+
query: request?.query
|
|
39460
|
+
},
|
|
39461
|
+
metadata: {
|
|
39462
|
+
...pickMetadata(request, RERANK_METADATA_ALLOWLIST),
|
|
39463
|
+
...Array.isArray(documents) ? { document_count: documents.length } : {}
|
|
39464
|
+
}
|
|
39465
|
+
};
|
|
39466
|
+
}
|
|
39467
|
+
function extractResponseMetadata3(result) {
|
|
39468
|
+
if (!isObject(result)) {
|
|
39469
|
+
return void 0;
|
|
39470
|
+
}
|
|
39471
|
+
const rawResponse = isObject(result.rawResponse) ? result.rawResponse : void 0;
|
|
39472
|
+
const model = typeof result.model === "string" ? result.model : typeof rawResponse?.model === "string" ? rawResponse.model : void 0;
|
|
39473
|
+
return model ? { model } : void 0;
|
|
39474
|
+
}
|
|
39475
|
+
function summarizeEmbeddingOutput(result) {
|
|
39476
|
+
return {
|
|
39477
|
+
count: isObject(result) && Array.isArray(result.data) ? result.data.length : 0
|
|
39478
|
+
};
|
|
39479
|
+
}
|
|
39480
|
+
function summarizeRerankOutput(result) {
|
|
39481
|
+
if (!isObject(result) || !Array.isArray(result.data)) {
|
|
39482
|
+
return void 0;
|
|
39483
|
+
}
|
|
39484
|
+
return result.data.slice(0, 100).map((item) => ({
|
|
39485
|
+
index: isObject(item) ? item.index : void 0,
|
|
39486
|
+
relevance_score: isObject(item) ? (typeof item.relevanceScore === "number" ? item.relevanceScore : item.relevance_score) ?? null : null
|
|
39487
|
+
}));
|
|
39488
|
+
}
|
|
39489
|
+
function summarizeContextualizedEmbeddingOutput(result) {
|
|
39490
|
+
if (!isObject(result)) {
|
|
39491
|
+
return { count: 0 };
|
|
39492
|
+
}
|
|
39493
|
+
if (Array.isArray(result.results)) {
|
|
39494
|
+
return {
|
|
39495
|
+
count: result.results.reduce(
|
|
39496
|
+
(count, item) => count + (isObject(item) && Array.isArray(item.embeddings) ? item.embeddings.length : 0),
|
|
39497
|
+
0
|
|
39498
|
+
)
|
|
39499
|
+
};
|
|
39500
|
+
}
|
|
39501
|
+
if (!Array.isArray(result.data)) {
|
|
39502
|
+
return { count: 0 };
|
|
39503
|
+
}
|
|
39504
|
+
return {
|
|
39505
|
+
count: result.data.reduce(
|
|
39506
|
+
(count, item) => count + (isObject(item) && Array.isArray(item.data) ? item.data.length : 0),
|
|
39507
|
+
0
|
|
39508
|
+
)
|
|
39509
|
+
};
|
|
39510
|
+
}
|
|
39511
|
+
function extractEmbeddingUsageMetrics(result) {
|
|
39512
|
+
const metrics = extractUsageMetrics3(result);
|
|
39513
|
+
return typeof metrics.tokens === "number" ? { prompt_tokens: metrics.tokens, tokens: metrics.tokens } : {};
|
|
39514
|
+
}
|
|
39515
|
+
function extractUsageMetrics3(result) {
|
|
39516
|
+
if (!isObject(result)) {
|
|
39517
|
+
return {};
|
|
39518
|
+
}
|
|
39519
|
+
const rawResponse = isObject(result.rawResponse) ? result.rawResponse : void 0;
|
|
39520
|
+
const usage = isObject(result.usage) ? result.usage : isObject(rawResponse?.usage) ? rawResponse.usage : void 0;
|
|
39521
|
+
const tokens = typeof result.totalTokens === "number" ? result.totalTokens : usage?.totalTokens ?? usage?.total_tokens;
|
|
39522
|
+
return typeof tokens === "number" && Number.isFinite(tokens) && tokens >= 0 ? { tokens } : {};
|
|
39523
|
+
}
|
|
39524
|
+
|
|
38713
39525
|
// src/instrumentation/plugins/cloudflare-ai-chat-channels.ts
|
|
38714
39526
|
var cloudflareAIChatChannels = defineChannels(
|
|
38715
39527
|
"@cloudflare/ai-chat",
|
|
@@ -39265,6 +40077,7 @@ var BraintrustPlugin = class extends BasePlugin {
|
|
|
39265
40077
|
langSmithPlugin = null;
|
|
39266
40078
|
piCodingAgentPlugin = null;
|
|
39267
40079
|
strandsAgentSDKPlugin = null;
|
|
40080
|
+
voyageAIPlugin = null;
|
|
39268
40081
|
cloudflareAIChatPlugin = null;
|
|
39269
40082
|
cloudflareAgentsPlugin = null;
|
|
39270
40083
|
constructor(config3 = {}) {
|
|
@@ -39339,6 +40152,10 @@ var BraintrustPlugin = class extends BasePlugin {
|
|
|
39339
40152
|
this.coherePlugin = new CoherePlugin();
|
|
39340
40153
|
this.coherePlugin.enable();
|
|
39341
40154
|
}
|
|
40155
|
+
if (integrations.voyageai !== false) {
|
|
40156
|
+
this.voyageAIPlugin = new VoyageAIPlugin();
|
|
40157
|
+
this.voyageAIPlugin.enable();
|
|
40158
|
+
}
|
|
39342
40159
|
if (integrations.groq !== false) {
|
|
39343
40160
|
this.groqPlugin = new GroqPlugin();
|
|
39344
40161
|
this.groqPlugin.enable();
|
|
@@ -39455,6 +40272,10 @@ var BraintrustPlugin = class extends BasePlugin {
|
|
|
39455
40272
|
this.coherePlugin.disable();
|
|
39456
40273
|
this.coherePlugin = null;
|
|
39457
40274
|
}
|
|
40275
|
+
if (this.voyageAIPlugin) {
|
|
40276
|
+
this.voyageAIPlugin.disable();
|
|
40277
|
+
this.voyageAIPlugin = null;
|
|
40278
|
+
}
|
|
39458
40279
|
if (this.groqPlugin) {
|
|
39459
40280
|
this.groqPlugin.disable();
|
|
39460
40281
|
this.groqPlugin = null;
|
|
@@ -39574,7 +40395,10 @@ var envIntegrationAliases = {
|
|
|
39574
40395
|
"langchain-js": "langchain",
|
|
39575
40396
|
"@langchain": "langchain",
|
|
39576
40397
|
langgraph: "langgraph",
|
|
39577
|
-
langsmith: "langsmith"
|
|
40398
|
+
langsmith: "langsmith",
|
|
40399
|
+
voyage: "voyageai",
|
|
40400
|
+
"voyage-ai": "voyageai",
|
|
40401
|
+
voyageai: "voyageai"
|
|
39578
40402
|
};
|
|
39579
40403
|
function getDefaultInstrumentationIntegrations() {
|
|
39580
40404
|
return {
|
|
@@ -39609,6 +40433,7 @@ function getDefaultInstrumentationIntegrations() {
|
|
|
39609
40433
|
langchain: true,
|
|
39610
40434
|
langgraph: true,
|
|
39611
40435
|
langsmith: true,
|
|
40436
|
+
voyageai: true,
|
|
39612
40437
|
piCodingAgent: true,
|
|
39613
40438
|
strandsAgentSDK: true,
|
|
39614
40439
|
cloudflareAgents: true
|
|
@@ -40029,9 +40854,9 @@ function configureNode() {
|
|
|
40029
40854
|
return value;
|
|
40030
40855
|
}
|
|
40031
40856
|
const envPaths = [];
|
|
40032
|
-
for (let dir2 = process.cwd(), depth = 0; depth <= BRAINTRUST_ENV_SEARCH_PARENT_LIMIT; dir2 =
|
|
40033
|
-
envPaths.push(
|
|
40034
|
-
if (
|
|
40857
|
+
for (let dir2 = process.cwd(), depth = 0; depth <= BRAINTRUST_ENV_SEARCH_PARENT_LIMIT; dir2 = path2.dirname(dir2), depth++) {
|
|
40858
|
+
envPaths.push(path2.join(dir2, ".env.braintrust"));
|
|
40859
|
+
if (path2.dirname(dir2) === dir2) {
|
|
40035
40860
|
break;
|
|
40036
40861
|
}
|
|
40037
40862
|
}
|
|
@@ -40073,10 +40898,10 @@ function configureNode() {
|
|
|
40073
40898
|
isomorph_default.processOn = (event, handler) => {
|
|
40074
40899
|
process.on(event, handler);
|
|
40075
40900
|
};
|
|
40076
|
-
isomorph_default.basename =
|
|
40901
|
+
isomorph_default.basename = path2.basename;
|
|
40077
40902
|
isomorph_default.writeln = (text) => process.stdout.write(text + "\n");
|
|
40078
|
-
isomorph_default.pathJoin =
|
|
40079
|
-
isomorph_default.pathDirname =
|
|
40903
|
+
isomorph_default.pathJoin = path2.join;
|
|
40904
|
+
isomorph_default.pathDirname = path2.dirname;
|
|
40080
40905
|
isomorph_default.mkdir = fs.mkdir;
|
|
40081
40906
|
isomorph_default.writeFile = fs.writeFile;
|
|
40082
40907
|
isomorph_default.readFile = fs.readFile;
|
|
@@ -40111,8 +40936,8 @@ function configureNode() {
|
|
|
40111
40936
|
registry.enable();
|
|
40112
40937
|
}
|
|
40113
40938
|
function getNearestBraintrustEnvValue(name) {
|
|
40114
|
-
for (let dir2 = process.cwd(), depth = 0; depth <= BRAINTRUST_ENV_SEARCH_PARENT_LIMIT; dir2 =
|
|
40115
|
-
const envPath =
|
|
40939
|
+
for (let dir2 = process.cwd(), depth = 0; depth <= BRAINTRUST_ENV_SEARCH_PARENT_LIMIT; dir2 = path2.dirname(dir2), depth++) {
|
|
40940
|
+
const envPath = path2.join(dir2, ".env.braintrust");
|
|
40116
40941
|
try {
|
|
40117
40942
|
const parsed = dotenv.parse(fsSync.readFileSync(envPath, "utf8"));
|
|
40118
40943
|
const value = parsed[name];
|
|
@@ -40122,7 +40947,7 @@ function getNearestBraintrustEnvValue(name) {
|
|
|
40122
40947
|
return void 0;
|
|
40123
40948
|
}
|
|
40124
40949
|
}
|
|
40125
|
-
if (
|
|
40950
|
+
if (path2.dirname(dir2) === dir2) {
|
|
40126
40951
|
break;
|
|
40127
40952
|
}
|
|
40128
40953
|
}
|
|
@@ -40133,7 +40958,7 @@ function getNearestBraintrustEnvValue(name) {
|
|
|
40133
40958
|
var import_env2 = require("@next/env");
|
|
40134
40959
|
|
|
40135
40960
|
// src/cli/functions/upload.ts
|
|
40136
|
-
var
|
|
40961
|
+
var import_node_fs3 = __toESM(require("node:fs"));
|
|
40137
40962
|
var import_node_path3 = __toESM(require("node:path"));
|
|
40138
40963
|
var import_node_zlib = require("node:zlib");
|
|
40139
40964
|
var import_v312 = require("zod/v3");
|
|
@@ -40143,11 +40968,11 @@ var import_source_map = require("source-map");
|
|
|
40143
40968
|
var fs2 = __toESM(require("node:fs/promises"));
|
|
40144
40969
|
|
|
40145
40970
|
// src/cli/jest/nodeModulesPaths.ts
|
|
40146
|
-
var
|
|
40147
|
-
var
|
|
40971
|
+
var path3 = __toESM(require("node:path"));
|
|
40972
|
+
var import_node_fs2 = require("node:fs");
|
|
40148
40973
|
function nodeModulesPaths(basedir, options) {
|
|
40149
40974
|
const modules = options && options.moduleDirectory ? Array.from(options.moduleDirectory) : ["node_modules"];
|
|
40150
|
-
const basedirAbs =
|
|
40975
|
+
const basedirAbs = path3.resolve(basedir);
|
|
40151
40976
|
let prefix = "/";
|
|
40152
40977
|
if (/^([A-Za-z]:)/.test(basedirAbs)) {
|
|
40153
40978
|
prefix = "";
|
|
@@ -40156,24 +40981,24 @@ function nodeModulesPaths(basedir, options) {
|
|
|
40156
40981
|
}
|
|
40157
40982
|
let physicalBasedir;
|
|
40158
40983
|
try {
|
|
40159
|
-
physicalBasedir =
|
|
40984
|
+
physicalBasedir = import_node_fs2.realpathSync.native(basedirAbs);
|
|
40160
40985
|
} catch {
|
|
40161
40986
|
physicalBasedir = basedirAbs;
|
|
40162
40987
|
}
|
|
40163
40988
|
const paths = [physicalBasedir];
|
|
40164
|
-
let parsed =
|
|
40989
|
+
let parsed = path3.parse(physicalBasedir);
|
|
40165
40990
|
while (parsed.dir !== paths[paths.length - 1]) {
|
|
40166
40991
|
paths.push(parsed.dir);
|
|
40167
|
-
parsed =
|
|
40992
|
+
parsed = path3.parse(parsed.dir);
|
|
40168
40993
|
}
|
|
40169
40994
|
const dirs = paths.reduce((dirs2, aPath) => {
|
|
40170
40995
|
for (const moduleDir of modules) {
|
|
40171
|
-
if (
|
|
40996
|
+
if (path3.isAbsolute(moduleDir)) {
|
|
40172
40997
|
if (aPath === basedirAbs && moduleDir) {
|
|
40173
40998
|
dirs2.push(moduleDir);
|
|
40174
40999
|
}
|
|
40175
41000
|
} else {
|
|
40176
|
-
dirs2.push(
|
|
41001
|
+
dirs2.push(path3.join(prefix, aPath, moduleDir));
|
|
40177
41002
|
}
|
|
40178
41003
|
}
|
|
40179
41004
|
return dirs2;
|
|
@@ -41163,7 +41988,7 @@ async function uploadBundles({
|
|
|
41163
41988
|
if (!pathInfo) {
|
|
41164
41989
|
return true;
|
|
41165
41990
|
}
|
|
41166
|
-
const bundleStream =
|
|
41991
|
+
const bundleStream = import_node_fs3.default.createReadStream(bundleFile).pipe((0, import_node_zlib.createGzip)());
|
|
41167
41992
|
const bundleData = await new Promise((resolve2, reject2) => {
|
|
41168
41993
|
const chunks = [];
|
|
41169
41994
|
bundleStream.on("data", (chunk) => {
|
|
@@ -41363,7 +42188,7 @@ async function bundleCommand(args) {
|
|
|
41363
42188
|
|
|
41364
42189
|
// src/cli/util/pull.ts
|
|
41365
42190
|
var import_v313 = require("zod/v3");
|
|
41366
|
-
var
|
|
42191
|
+
var import_promises2 = __toESM(require("node:fs/promises"));
|
|
41367
42192
|
var import_node_util4 = __toESM(require("node:util"));
|
|
41368
42193
|
var import_node_path4 = __toESM(require("node:path"));
|
|
41369
42194
|
var import_pluralize3 = __toESM(require("pluralize"));
|
|
@@ -41401,14 +42226,13 @@ async function pullCommand(args) {
|
|
|
41401
42226
|
console.log(` * ${projectName}`);
|
|
41402
42227
|
}
|
|
41403
42228
|
const outputDir = args.output_dir ?? "./braintrust";
|
|
41404
|
-
await
|
|
41405
|
-
const
|
|
41406
|
-
const
|
|
41407
|
-
|
|
42229
|
+
await import_promises2.default.mkdir(outputDir, { recursive: true });
|
|
42230
|
+
const repoRoot = await currentRepoPath();
|
|
42231
|
+
const dirtyFileOutput = repoRoot ? await runGitCommand(["diff", "--name-only", "-z", "HEAD"], {
|
|
42232
|
+
cwd: repoRoot
|
|
42233
|
+
}) : "";
|
|
41408
42234
|
const dirtyFiles = new Set(
|
|
41409
|
-
(
|
|
41410
|
-
(f) => import_node_path4.default.resolve(repoRoot ?? ".", f.file)
|
|
41411
|
-
)
|
|
42235
|
+
dirtyFileOutput.split("\0").filter(Boolean).map((file) => import_node_path4.default.resolve(repoRoot ?? ".", file))
|
|
41412
42236
|
);
|
|
41413
42237
|
for (const projectName of Object.keys(projectNameToFunctions)) {
|
|
41414
42238
|
const projectFile = import_node_path4.default.join(
|
|
@@ -41416,7 +42240,7 @@ async function pullCommand(args) {
|
|
|
41416
42240
|
`${slugify(projectName, { lower: true, strict: true, trim: true })}.ts`
|
|
41417
42241
|
);
|
|
41418
42242
|
const resolvedProjectFile = import_node_path4.default.resolve(projectFile);
|
|
41419
|
-
const fileExists = await
|
|
42243
|
+
const fileExists = await import_promises2.default.stat(projectFile).then(
|
|
41420
42244
|
() => true,
|
|
41421
42245
|
() => false
|
|
41422
42246
|
);
|
|
@@ -41436,7 +42260,7 @@ async function pullCommand(args) {
|
|
|
41436
42260
|
);
|
|
41437
42261
|
continue;
|
|
41438
42262
|
} else if (fileExists) {
|
|
41439
|
-
if (!
|
|
42263
|
+
if (!repoRoot) {
|
|
41440
42264
|
console.warn(
|
|
41441
42265
|
warning(
|
|
41442
42266
|
`Project ${projectName} already exists in ${doubleQuote(projectFile)}. Skipping since this is not a git repository...`
|
|
@@ -41458,7 +42282,7 @@ async function pullCommand(args) {
|
|
|
41458
42282
|
functions: projectNameToFunctions[projectName],
|
|
41459
42283
|
hasSpecifiedFunction: !!args.slug || !!args.id
|
|
41460
42284
|
});
|
|
41461
|
-
await
|
|
42285
|
+
await import_promises2.default.writeFile(projectFile, projectFileContents || "");
|
|
41462
42286
|
console.log(`Wrote ${projectName} to ${doubleQuote(projectFile)}`);
|
|
41463
42287
|
}
|
|
41464
42288
|
}
|
|
@@ -42151,6 +42975,8 @@ async function getDataset(state, data) {
|
|
|
42151
42975
|
environment: data.dataset_environment ?? void 0,
|
|
42152
42976
|
_internal_btql: data._internal_btql ?? void 0
|
|
42153
42977
|
});
|
|
42978
|
+
} else if ("experiment_name" in data) {
|
|
42979
|
+
return BaseExperiment({ name: data.experiment_name });
|
|
42154
42980
|
} else {
|
|
42155
42981
|
return data.data;
|
|
42156
42982
|
}
|
|
@@ -42676,7 +43502,7 @@ function checkMatch(pathInput, include_patterns, exclude_patterns) {
|
|
|
42676
43502
|
async function collectFiles(inputPath, mode) {
|
|
42677
43503
|
let pathStat = null;
|
|
42678
43504
|
try {
|
|
42679
|
-
pathStat =
|
|
43505
|
+
pathStat = import_node_fs4.default.lstatSync(inputPath);
|
|
42680
43506
|
} catch (e) {
|
|
42681
43507
|
console.error(error(`Error reading ${inputPath}: ${e}`));
|
|
42682
43508
|
process.exit(1);
|
|
@@ -42732,8 +43558,8 @@ var nativeNodeModulesPlugin = {
|
|
|
42732
43558
|
};
|
|
42733
43559
|
build2.onResolve({ filter: /\.node$/ }, (args) => {
|
|
42734
43560
|
try {
|
|
42735
|
-
const
|
|
42736
|
-
const match =
|
|
43561
|
+
const path9 = require.resolve(args.path, { paths: [args.resolveDir] });
|
|
43562
|
+
const match = path9.match(
|
|
42737
43563
|
/node_modules[/\\]((?:@[^/\\]+[/\\])?[^/\\]+)/
|
|
42738
43564
|
);
|
|
42739
43565
|
if (match) {
|