braintrust 3.27.0 → 3.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dev/dist/index.d.mts +519 -186
- package/dev/dist/index.d.ts +519 -186
- package/dev/dist/index.js +1837 -1009
- package/dev/dist/index.mjs +1172 -344
- package/dist/apply-auto-instrumentation.js +262 -210
- package/dist/apply-auto-instrumentation.mjs +54 -2
- package/dist/auto-instrumentations/bundler/esbuild.cjs +81 -2
- package/dist/auto-instrumentations/bundler/esbuild.mjs +2 -2
- package/dist/auto-instrumentations/bundler/next.cjs +81 -2
- package/dist/auto-instrumentations/bundler/next.mjs +3 -3
- package/dist/auto-instrumentations/bundler/rollup.cjs +81 -2
- package/dist/auto-instrumentations/bundler/rollup.mjs +2 -2
- package/dist/auto-instrumentations/bundler/vite.cjs +81 -2
- package/dist/auto-instrumentations/bundler/vite.mjs +2 -2
- package/dist/auto-instrumentations/bundler/webpack-loader.cjs +81 -2
- package/dist/auto-instrumentations/bundler/webpack.cjs +81 -2
- package/dist/auto-instrumentations/bundler/webpack.mjs +3 -3
- package/dist/auto-instrumentations/{chunk-XEYKUBLY.mjs → chunk-26PKVUKB.mjs} +80 -2
- package/dist/auto-instrumentations/{chunk-BW33ULMW.mjs → chunk-HD35AM3M.mjs} +1 -1
- package/dist/auto-instrumentations/{chunk-ZNHTSSGI.mjs → chunk-NP7V4XB2.mjs} +2 -1
- package/dist/auto-instrumentations/hook.mjs +236 -30
- package/dist/auto-instrumentations/index.cjs +2 -1
- package/dist/auto-instrumentations/index.mjs +1 -1
- package/dist/browser.d.mts +628 -53
- package/dist/browser.d.ts +628 -53
- package/dist/browser.js +2301 -284
- package/dist/browser.mjs +2301 -284
- package/dist/{chunk-MF7NU6BT.js → chunk-BBE7SNRV.js} +34 -4
- package/dist/{chunk-QRHGVBKU.js → chunk-OBBWQW6K.js} +1799 -1023
- package/dist/{chunk-YKD22IMR.mjs → chunk-UPFNQCGB.mjs} +966 -190
- package/dist/{chunk-CZM5JIQL.mjs → chunk-ZHUHZWFY.mjs} +33 -3
- package/dist/cli.js +1224 -398
- package/dist/edge-light.d.mts +1 -1
- package/dist/edge-light.d.ts +1 -1
- package/dist/edge-light.js +2301 -284
- package/dist/edge-light.mjs +2301 -284
- package/dist/index.d.mts +1212 -637
- package/dist/index.d.ts +1212 -637
- package/dist/index.js +1880 -590
- package/dist/index.mjs +1450 -160
- package/dist/instrumentation/index.d.mts +190 -6
- package/dist/instrumentation/index.d.ts +190 -6
- package/dist/instrumentation/index.js +823 -116
- package/dist/instrumentation/index.mjs +823 -116
- package/dist/vitest-evals-reporter.js +16 -16
- package/dist/vitest-evals-reporter.mjs +2 -2
- package/dist/workerd.d.mts +1 -1
- package/dist/workerd.d.ts +1 -1
- package/dist/workerd.js +2301 -284
- package/dist/workerd.mjs +2301 -284
- package/package.json +2 -3
- package/util/dist/index.d.mts +1545 -100
- package/util/dist/index.d.ts +1545 -100
package/dev/dist/index.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// src/node/config.ts
|
|
2
2
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
3
|
-
import * as
|
|
3
|
+
import * as path2 from "node:path";
|
|
4
4
|
import * as fs from "node:fs/promises";
|
|
5
5
|
import * as os from "node:os";
|
|
6
6
|
import * as fsSync from "node:fs";
|
|
@@ -800,34 +800,93 @@ setGlobalHookErrorReporter((error) => {
|
|
|
800
800
|
debugLogger.error("Global instrumentation hook error:", error);
|
|
801
801
|
});
|
|
802
802
|
|
|
803
|
-
// src/
|
|
804
|
-
import {
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
803
|
+
// src/git-command.ts
|
|
804
|
+
import { execFile } from "node:child_process";
|
|
805
|
+
import { constants } from "node:fs";
|
|
806
|
+
import { access } from "node:fs/promises";
|
|
807
|
+
import * as path from "node:path";
|
|
808
|
+
var GIT_EXECUTABLE_NAMES = process.platform === "win32" ? ["git.exe", "git"] : ["git"];
|
|
809
|
+
var GIT_MAX_BUFFER_BYTES = 10 * 1024 * 1024;
|
|
810
|
+
var gitExecutablePromise;
|
|
811
|
+
function executableSearchPath() {
|
|
812
|
+
return Object.entries(process.env).find(
|
|
813
|
+
([name]) => name.toUpperCase() === "PATH"
|
|
814
|
+
)?.[1];
|
|
815
|
+
}
|
|
816
|
+
async function findGitExecutable(searchPath = executableSearchPath()) {
|
|
817
|
+
if (!searchPath) {
|
|
818
|
+
return void 0;
|
|
819
|
+
}
|
|
820
|
+
for (const rawSearchDir of searchPath.split(path.delimiter)) {
|
|
821
|
+
const searchDir = rawSearchDir.trim().replace(/^"(.*)"$/, "$1");
|
|
822
|
+
if (!path.isAbsolute(searchDir)) {
|
|
823
|
+
continue;
|
|
824
|
+
}
|
|
825
|
+
for (const executableName of GIT_EXECUTABLE_NAMES) {
|
|
826
|
+
const candidate = path.join(searchDir, executableName);
|
|
827
|
+
try {
|
|
828
|
+
await access(
|
|
829
|
+
candidate,
|
|
830
|
+
process.platform === "win32" ? constants.F_OK : constants.X_OK
|
|
831
|
+
);
|
|
832
|
+
return candidate;
|
|
833
|
+
} catch {
|
|
834
|
+
}
|
|
813
835
|
}
|
|
814
|
-
} catch {
|
|
815
|
-
return null;
|
|
816
836
|
}
|
|
837
|
+
return void 0;
|
|
817
838
|
}
|
|
839
|
+
async function resolveGitExecutable() {
|
|
840
|
+
gitExecutablePromise ??= findGitExecutable();
|
|
841
|
+
return await gitExecutablePromise;
|
|
842
|
+
}
|
|
843
|
+
async function runGitCommand(args, options = {}) {
|
|
844
|
+
const executable = await resolveGitExecutable();
|
|
845
|
+
if (!executable) {
|
|
846
|
+
throw new Error("Could not find a git executable on PATH");
|
|
847
|
+
}
|
|
848
|
+
return await new Promise((resolve, reject2) => {
|
|
849
|
+
execFile(
|
|
850
|
+
executable,
|
|
851
|
+
args,
|
|
852
|
+
{
|
|
853
|
+
cwd: options.cwd,
|
|
854
|
+
encoding: "utf8",
|
|
855
|
+
maxBuffer: GIT_MAX_BUFFER_BYTES
|
|
856
|
+
},
|
|
857
|
+
(error, stdout) => {
|
|
858
|
+
if (error) {
|
|
859
|
+
reject2(error);
|
|
860
|
+
} else {
|
|
861
|
+
resolve(stdout);
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
);
|
|
865
|
+
});
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
// src/gitutil.ts
|
|
869
|
+
var COMMON_BASE_BRANCHES = ["main", "master", "develop"];
|
|
818
870
|
var _baseBranch = null;
|
|
819
871
|
async function getBaseBranch(remote = void 0) {
|
|
820
872
|
if (_baseBranch === null) {
|
|
821
|
-
const
|
|
822
|
-
if (
|
|
873
|
+
const repoPath = await currentRepoPath();
|
|
874
|
+
if (!repoPath) {
|
|
823
875
|
throw new Error("Not in a git repo");
|
|
824
876
|
}
|
|
825
|
-
const
|
|
877
|
+
const runGit = async (args) => await runGitCommand(args, { cwd: repoPath });
|
|
878
|
+
const remoteName = remote ?? (await runGit(["remote"])).trim().split(/\r?\n/)[0];
|
|
826
879
|
if (!remoteName) {
|
|
827
880
|
throw new Error("No remote found");
|
|
828
881
|
}
|
|
829
882
|
let branch = null;
|
|
830
|
-
const repoBranches = new Set(
|
|
883
|
+
const repoBranches = new Set(
|
|
884
|
+
(await runGit([
|
|
885
|
+
"for-each-ref",
|
|
886
|
+
"--format=%(refname:short)",
|
|
887
|
+
"refs/heads/"
|
|
888
|
+
])).trim().split(/\r?\n/)
|
|
889
|
+
);
|
|
831
890
|
const matchingBaseBranches = COMMON_BASE_BRANCHES.filter(
|
|
832
891
|
(b) => repoBranches.has(b)
|
|
833
892
|
);
|
|
@@ -835,7 +894,7 @@ async function getBaseBranch(remote = void 0) {
|
|
|
835
894
|
branch = matchingBaseBranches[0];
|
|
836
895
|
} else {
|
|
837
896
|
try {
|
|
838
|
-
const remoteInfo = await
|
|
897
|
+
const remoteInfo = await runGit(["remote", "show", remoteName]);
|
|
839
898
|
if (!remoteInfo) {
|
|
840
899
|
throw new Error(`Could not find remote ${remoteName}`);
|
|
841
900
|
}
|
|
@@ -853,27 +912,28 @@ async function getBaseBranch(remote = void 0) {
|
|
|
853
912
|
return _baseBranch;
|
|
854
913
|
}
|
|
855
914
|
async function getBaseBranchAncestor(remote = void 0) {
|
|
856
|
-
const
|
|
857
|
-
if (
|
|
915
|
+
const repoPath = await currentRepoPath();
|
|
916
|
+
if (!repoPath) {
|
|
858
917
|
throw new Error("Not in a git repo");
|
|
859
918
|
}
|
|
860
919
|
const { remote: remoteName, branch: baseBranch } = await getBaseBranch(remote);
|
|
861
|
-
const isDirty = (await
|
|
920
|
+
const isDirty = (await runGitCommand(["diff", "--name-only"], {
|
|
921
|
+
cwd: repoPath
|
|
922
|
+
})).trim().length > 0;
|
|
862
923
|
const head = isDirty ? "HEAD" : "HEAD^";
|
|
863
924
|
try {
|
|
864
|
-
const ancestor = await
|
|
865
|
-
"merge-base",
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
]);
|
|
925
|
+
const ancestor = await runGitCommand(
|
|
926
|
+
["merge-base", head, `${remoteName}/${baseBranch}`],
|
|
927
|
+
{ cwd: repoPath }
|
|
928
|
+
);
|
|
869
929
|
return ancestor.trim();
|
|
870
930
|
} catch {
|
|
871
931
|
return void 0;
|
|
872
932
|
}
|
|
873
933
|
}
|
|
874
934
|
async function getPastNAncestors(n = 1e3, remote = void 0) {
|
|
875
|
-
const
|
|
876
|
-
if (
|
|
935
|
+
const repoPath = await currentRepoPath();
|
|
936
|
+
if (!repoPath) {
|
|
877
937
|
return [];
|
|
878
938
|
}
|
|
879
939
|
let ancestor = void 0;
|
|
@@ -888,8 +948,10 @@ async function getPastNAncestors(n = 1e3, remote = void 0) {
|
|
|
888
948
|
if (!ancestor) {
|
|
889
949
|
return [];
|
|
890
950
|
}
|
|
891
|
-
const commits = await
|
|
892
|
-
|
|
951
|
+
const commits = (await runGitCommand(["rev-list", `--max-count=${n}`, `${ancestor}..HEAD`], {
|
|
952
|
+
cwd: repoPath
|
|
953
|
+
})).trim();
|
|
954
|
+
return commits ? commits.split(/\r?\n/).slice(0, n) : [];
|
|
893
955
|
}
|
|
894
956
|
async function attempt(fn) {
|
|
895
957
|
try {
|
|
@@ -920,9 +982,14 @@ async function getRepoInfo(settings) {
|
|
|
920
982
|
});
|
|
921
983
|
return sanitized;
|
|
922
984
|
}
|
|
985
|
+
async function currentRepoPath() {
|
|
986
|
+
return await attempt(
|
|
987
|
+
async () => (await runGitCommand(["rev-parse", "--show-toplevel"])).trim()
|
|
988
|
+
);
|
|
989
|
+
}
|
|
923
990
|
async function repoInfo() {
|
|
924
|
-
const
|
|
925
|
-
if (
|
|
991
|
+
const repoPath = await currentRepoPath();
|
|
992
|
+
if (!repoPath) {
|
|
926
993
|
return void 0;
|
|
927
994
|
}
|
|
928
995
|
let commit = void 0;
|
|
@@ -933,29 +1000,32 @@ async function repoInfo() {
|
|
|
933
1000
|
let tag = void 0;
|
|
934
1001
|
let branch = void 0;
|
|
935
1002
|
let git_diff = void 0;
|
|
936
|
-
const
|
|
937
|
-
|
|
1003
|
+
const runGit = async (args) => await runGitCommand(args, { cwd: repoPath });
|
|
1004
|
+
const dirty = (await runGit(["diff", "--name-only"])).trim().length > 0;
|
|
1005
|
+
commit = await attempt(
|
|
1006
|
+
async () => (await runGit(["rev-parse", "HEAD"])).trim()
|
|
1007
|
+
);
|
|
938
1008
|
commit_message = await attempt(
|
|
939
|
-
async () => (await
|
|
1009
|
+
async () => (await runGit(["log", "-1", "--pretty=%B"])).trim()
|
|
940
1010
|
);
|
|
941
1011
|
commit_time = await attempt(
|
|
942
|
-
async () => (await
|
|
1012
|
+
async () => (await runGit(["log", "-1", "--pretty=%cI"])).trim()
|
|
943
1013
|
);
|
|
944
1014
|
author_name = await attempt(
|
|
945
|
-
async () => (await
|
|
1015
|
+
async () => (await runGit(["log", "-1", "--pretty=%aN"])).trim()
|
|
946
1016
|
);
|
|
947
1017
|
author_email = await attempt(
|
|
948
|
-
async () => (await
|
|
1018
|
+
async () => (await runGit(["log", "-1", "--pretty=%aE"])).trim()
|
|
949
1019
|
);
|
|
950
1020
|
tag = await attempt(
|
|
951
|
-
async () => (await
|
|
1021
|
+
async () => (await runGit(["describe", "--tags", "--exact-match", "--always"])).trim()
|
|
952
1022
|
);
|
|
953
1023
|
branch = await attempt(
|
|
954
|
-
async () => (await
|
|
1024
|
+
async () => (await runGit(["rev-parse", "--abbrev-ref", "HEAD"])).trim()
|
|
955
1025
|
);
|
|
956
1026
|
if (dirty) {
|
|
957
1027
|
git_diff = await attempt(
|
|
958
|
-
async () => truncateToByteLimit(await
|
|
1028
|
+
async () => truncateToByteLimit(await runGit(["diff", "--no-ext-diff", "HEAD"]))
|
|
959
1029
|
);
|
|
960
1030
|
}
|
|
961
1031
|
return {
|
|
@@ -2036,15 +2106,15 @@ function mergeDictsWithPaths({
|
|
|
2036
2106
|
function mergeDictsWithPathsHelper({
|
|
2037
2107
|
mergeInto,
|
|
2038
2108
|
mergeFrom,
|
|
2039
|
-
path:
|
|
2109
|
+
path: path3,
|
|
2040
2110
|
mergePaths
|
|
2041
2111
|
}) {
|
|
2042
2112
|
Object.entries(mergeFrom).forEach(([k, mergeFromV]) => {
|
|
2043
2113
|
if (FORBIDDEN_MERGE_KEYS.has(k)) return;
|
|
2044
|
-
const fullPath =
|
|
2114
|
+
const fullPath = path3.concat([k]);
|
|
2045
2115
|
const fullPathSerialized = JSON.stringify(fullPath);
|
|
2046
2116
|
const mergeIntoV = recordFind(mergeInto, k);
|
|
2047
|
-
const isSetUnionField =
|
|
2117
|
+
const isSetUnionField = path3.length === 0 && SET_UNION_FIELDS.has(k) && !mergePaths.has(fullPathSerialized);
|
|
2048
2118
|
if (isSetUnionField && isArray(mergeIntoV) && isArray(mergeFromV)) {
|
|
2049
2119
|
const seen = /* @__PURE__ */ new Set();
|
|
2050
2120
|
const combined = [];
|
|
@@ -2077,9 +2147,9 @@ function mergeDicts(mergeInto, mergeFrom) {
|
|
|
2077
2147
|
function recordFind(m, k) {
|
|
2078
2148
|
return m[k];
|
|
2079
2149
|
}
|
|
2080
|
-
function getObjValueByPath(row,
|
|
2150
|
+
function getObjValueByPath(row, path3) {
|
|
2081
2151
|
let curr = row;
|
|
2082
|
-
for (const p of
|
|
2152
|
+
for (const p of path3) {
|
|
2083
2153
|
if (!isObjectOrArray(curr)) {
|
|
2084
2154
|
return null;
|
|
2085
2155
|
}
|
|
@@ -2706,7 +2776,8 @@ var AclObjectType = z6.union([
|
|
|
2706
2776
|
"org_member",
|
|
2707
2777
|
"project_log",
|
|
2708
2778
|
"org_project",
|
|
2709
|
-
"org_audit_logs"
|
|
2779
|
+
"org_audit_logs",
|
|
2780
|
+
"project_group"
|
|
2710
2781
|
]),
|
|
2711
2782
|
z6.null()
|
|
2712
2783
|
]);
|
|
@@ -2835,7 +2906,8 @@ var AsyncScoringState = z6.union([
|
|
|
2835
2906
|
token: z6.string(),
|
|
2836
2907
|
function_ids: z6.array(z6.unknown()),
|
|
2837
2908
|
skip_logging: z6.union([z6.boolean(), z6.null()]).optional(),
|
|
2838
|
-
triggered_functions: z6.union([z6.record(TriggeredFunctionState), z6.null()]).optional()
|
|
2909
|
+
triggered_functions: z6.union([z6.record(TriggeredFunctionState), z6.null()]).optional(),
|
|
2910
|
+
last_triggered_xact_id: z6.union([z6.string(), z6.number(), z6.null()]).optional()
|
|
2839
2911
|
}),
|
|
2840
2912
|
z6.object({ status: z6.literal("disabled") }),
|
|
2841
2913
|
z6.null(),
|
|
@@ -2904,7 +2976,7 @@ var FunctionTypeEnum = z6.enum([
|
|
|
2904
2976
|
"parameters",
|
|
2905
2977
|
"sandbox"
|
|
2906
2978
|
]);
|
|
2907
|
-
var
|
|
2979
|
+
var FacetPreprocessorId = z6.union([
|
|
2908
2980
|
z6.object({
|
|
2909
2981
|
type: z6.literal("function"),
|
|
2910
2982
|
id: z6.string(),
|
|
@@ -2915,10 +2987,23 @@ var NullableSavedFunctionId = z6.union([
|
|
|
2915
2987
|
name: z6.string(),
|
|
2916
2988
|
function_type: FunctionTypeEnum.optional().default("scorer")
|
|
2917
2989
|
}),
|
|
2990
|
+
z6.object({ type: z6.literal("inline"), code: z6.string().min(1) }),
|
|
2918
2991
|
z6.null()
|
|
2919
2992
|
]);
|
|
2993
|
+
var SavedFunctionId = z6.union([
|
|
2994
|
+
z6.object({
|
|
2995
|
+
type: z6.literal("function"),
|
|
2996
|
+
id: z6.string(),
|
|
2997
|
+
version: z6.string().optional()
|
|
2998
|
+
}),
|
|
2999
|
+
z6.object({
|
|
3000
|
+
type: z6.literal("global"),
|
|
3001
|
+
name: z6.string(),
|
|
3002
|
+
function_type: FunctionTypeEnum.optional().default("scorer")
|
|
3003
|
+
})
|
|
3004
|
+
]);
|
|
2920
3005
|
var TopicMapGenerationSettings = z6.object({
|
|
2921
|
-
algorithm: z6.enum(["hdbscan", "kmeans"]),
|
|
3006
|
+
algorithm: z6.enum(["hdbscan", "kmeans", "community"]),
|
|
2922
3007
|
dimension_reduction: z6.enum(["umap", "pca", "none"]),
|
|
2923
3008
|
sample_size: z6.number().int().gt(0).optional(),
|
|
2924
3009
|
n_clusters: z6.number().int().gt(0).optional(),
|
|
@@ -2930,6 +3015,7 @@ var TopicMapGenerationSettings = z6.object({
|
|
|
2930
3015
|
var TopicMapData = z6.object({
|
|
2931
3016
|
type: z6.literal("topic_map"),
|
|
2932
3017
|
source_facet: z6.string(),
|
|
3018
|
+
source_facet_function: SavedFunctionId.and(z6.unknown()).optional(),
|
|
2933
3019
|
embedding_model: z6.string(),
|
|
2934
3020
|
bundle_key: z6.string().optional(),
|
|
2935
3021
|
report_key: z6.string().optional(),
|
|
@@ -2943,7 +3029,7 @@ var TopicMapData = z6.object({
|
|
|
2943
3029
|
});
|
|
2944
3030
|
var BatchedFacetData = z6.object({
|
|
2945
3031
|
type: z6.literal("batched_facet"),
|
|
2946
|
-
preprocessor:
|
|
3032
|
+
preprocessor: FacetPreprocessorId.optional(),
|
|
2947
3033
|
facets: z6.array(
|
|
2948
3034
|
z6.object({
|
|
2949
3035
|
name: z6.string(),
|
|
@@ -3204,18 +3290,6 @@ var ObjectReferenceNullish = z6.union([
|
|
|
3204
3290
|
}),
|
|
3205
3291
|
z6.null()
|
|
3206
3292
|
]);
|
|
3207
|
-
var SavedFunctionId = z6.union([
|
|
3208
|
-
z6.object({
|
|
3209
|
-
type: z6.literal("function"),
|
|
3210
|
-
id: z6.string(),
|
|
3211
|
-
version: z6.string().optional()
|
|
3212
|
-
}),
|
|
3213
|
-
z6.object({
|
|
3214
|
-
type: z6.literal("global"),
|
|
3215
|
-
name: z6.string(),
|
|
3216
|
-
function_type: FunctionTypeEnum.optional().default("scorer")
|
|
3217
|
-
})
|
|
3218
|
-
]);
|
|
3219
3293
|
var DatasetEvent = z6.object({
|
|
3220
3294
|
id: z6.string(),
|
|
3221
3295
|
_xact_id: z6.string(),
|
|
@@ -3437,7 +3511,7 @@ var ExtendedSavedFunctionId = z6.union([
|
|
|
3437
3511
|
]);
|
|
3438
3512
|
var FacetData = z6.object({
|
|
3439
3513
|
type: z6.literal("facet"),
|
|
3440
|
-
preprocessor:
|
|
3514
|
+
preprocessor: FacetPreprocessorId.optional(),
|
|
3441
3515
|
prompt: z6.string(),
|
|
3442
3516
|
model: z6.string().optional(),
|
|
3443
3517
|
embedding_model: z6.string().optional(),
|
|
@@ -3942,6 +4016,19 @@ var MessageRole = z6.enum([
|
|
|
3942
4016
|
"model",
|
|
3943
4017
|
"developer"
|
|
3944
4018
|
]);
|
|
4019
|
+
var NullableSavedFunctionId = z6.union([
|
|
4020
|
+
z6.object({
|
|
4021
|
+
type: z6.literal("function"),
|
|
4022
|
+
id: z6.string(),
|
|
4023
|
+
version: z6.string().optional()
|
|
4024
|
+
}),
|
|
4025
|
+
z6.object({
|
|
4026
|
+
type: z6.literal("global"),
|
|
4027
|
+
name: z6.string(),
|
|
4028
|
+
function_type: FunctionTypeEnum.optional().default("scorer")
|
|
4029
|
+
}),
|
|
4030
|
+
z6.null()
|
|
4031
|
+
]);
|
|
3945
4032
|
var ObjectReference = z6.object({
|
|
3946
4033
|
object_type: z6.enum([
|
|
3947
4034
|
"project_logs",
|
|
@@ -3963,6 +4050,7 @@ var TraceScope = z6.object({
|
|
|
3963
4050
|
});
|
|
3964
4051
|
var OnlineScoreConfig = z6.union([
|
|
3965
4052
|
z6.object({
|
|
4053
|
+
status: AutomationStatus.optional(),
|
|
3966
4054
|
sampling_rate: z6.number().gte(0).lte(1),
|
|
3967
4055
|
scorers: z6.array(SavedFunctionId),
|
|
3968
4056
|
btql_filter: z6.union([z6.string(), z6.null()]).optional(),
|
|
@@ -4043,6 +4131,72 @@ var Project = z6.object({
|
|
|
4043
4131
|
user_id: z6.union([z6.string(), z6.null()]).optional(),
|
|
4044
4132
|
settings: ProjectSettings.optional()
|
|
4045
4133
|
});
|
|
4134
|
+
var WindowedAutomationConfig = z6.object({
|
|
4135
|
+
event_type: z6.literal("windowed"),
|
|
4136
|
+
product_origin: z6.union([z6.literal("patterns"), z6.null()]).optional(),
|
|
4137
|
+
status: AutomationStatus.optional(),
|
|
4138
|
+
threshold: z6.object({
|
|
4139
|
+
calculation: z6.object({
|
|
4140
|
+
type: z6.literal("btql"),
|
|
4141
|
+
btql_query: z6.string().min(1),
|
|
4142
|
+
output: z6.object({
|
|
4143
|
+
type: z6.literal("scalar"),
|
|
4144
|
+
value_column: z6.string().min(1)
|
|
4145
|
+
})
|
|
4146
|
+
}),
|
|
4147
|
+
policy: z6.object({
|
|
4148
|
+
condition: z6.object({
|
|
4149
|
+
type: z6.literal("threshold"),
|
|
4150
|
+
operator: z6.enum(["lt", "lte", "gt", "gte", "eq", "neq"]),
|
|
4151
|
+
threshold: z6.number()
|
|
4152
|
+
}),
|
|
4153
|
+
pending_seconds: z6.number().int().gte(0).lte(2592e3),
|
|
4154
|
+
no_data_behavior: z6.enum(["keep_last", "resolve", "alert"]),
|
|
4155
|
+
renotify_interval_seconds: z6.union([z6.number(), z6.null()]).optional(),
|
|
4156
|
+
notify_on_recovery: z6.boolean().optional().default(true)
|
|
4157
|
+
})
|
|
4158
|
+
}).optional(),
|
|
4159
|
+
window: z6.object({
|
|
4160
|
+
window_seconds: z6.number().int().gte(1).lte(2592e3),
|
|
4161
|
+
schedule: z6.union([
|
|
4162
|
+
z6.object({
|
|
4163
|
+
type: z6.literal("interval"),
|
|
4164
|
+
evaluation_interval_seconds: z6.number().int().gte(1).lte(2592e3)
|
|
4165
|
+
}),
|
|
4166
|
+
z6.object({
|
|
4167
|
+
type: z6.literal("cron"),
|
|
4168
|
+
cron_expression: z6.string().min(1),
|
|
4169
|
+
timezone: z6.union([z6.string(), z6.null()]).optional()
|
|
4170
|
+
})
|
|
4171
|
+
]),
|
|
4172
|
+
evaluation_delay_seconds: z6.number().int().gte(0).lte(2592e3)
|
|
4173
|
+
}),
|
|
4174
|
+
loop: z6.object({
|
|
4175
|
+
prompt: z6.string().min(1).max(1e4),
|
|
4176
|
+
include_trigger_input: z6.boolean().optional().default(false),
|
|
4177
|
+
agent_slug: z6.string().min(1),
|
|
4178
|
+
auto_approve_tools: z6.array(z6.string().min(1)).optional().default([]),
|
|
4179
|
+
harness: z6.enum(["native", "codex", "claude-code"]).optional(),
|
|
4180
|
+
model: z6.string().min(1).optional(),
|
|
4181
|
+
reasoning_effort: z6.enum(["none", "minimal", "low", "medium", "high", "xhigh", "max"]).optional()
|
|
4182
|
+
}).optional(),
|
|
4183
|
+
actions: z6.array(
|
|
4184
|
+
z6.union([
|
|
4185
|
+
z6.object({
|
|
4186
|
+
type: z6.literal("webhook"),
|
|
4187
|
+
url: z6.string(),
|
|
4188
|
+
formatting_prompt: z6.string().min(1).max(1e4).optional()
|
|
4189
|
+
}),
|
|
4190
|
+
z6.object({
|
|
4191
|
+
type: z6.literal("slack"),
|
|
4192
|
+
workspace_id: z6.string(),
|
|
4193
|
+
channel: z6.string(),
|
|
4194
|
+
message_template: z6.string().optional(),
|
|
4195
|
+
formatting_prompt: z6.string().min(1).max(1e4).optional()
|
|
4196
|
+
})
|
|
4197
|
+
])
|
|
4198
|
+
).max(20).optional().default([])
|
|
4199
|
+
});
|
|
4046
4200
|
var TopicAutomationFacetModel = z6.union([
|
|
4047
4201
|
z6.enum(["brain-facet-latest", "brain-facet-1", "brain-facet-2"]),
|
|
4048
4202
|
z6.null()
|
|
@@ -4084,7 +4238,8 @@ var TopicDigestAutomationConfig = z6.object({
|
|
|
4084
4238
|
type: z6.literal("slack"),
|
|
4085
4239
|
workspace_id: z6.string(),
|
|
4086
4240
|
channel: z6.string(),
|
|
4087
|
-
message_template: z6.string().optional()
|
|
4241
|
+
message_template: z6.string().optional(),
|
|
4242
|
+
formatting_prompt: z6.string().min(1).max(1e4).optional()
|
|
4088
4243
|
}),
|
|
4089
4244
|
topic_map_function_ids: z6.array(z6.string()).max(10).optional()
|
|
4090
4245
|
});
|
|
@@ -4098,15 +4253,21 @@ var ProjectAutomation = z6.object({
|
|
|
4098
4253
|
config: z6.union([
|
|
4099
4254
|
z6.object({
|
|
4100
4255
|
event_type: z6.literal("logs"),
|
|
4256
|
+
status: AutomationStatus.optional(),
|
|
4101
4257
|
btql_filter: z6.string(),
|
|
4102
4258
|
interval_seconds: z6.number().gte(1).lte(2592e3),
|
|
4103
4259
|
action: z6.union([
|
|
4104
|
-
z6.object({
|
|
4260
|
+
z6.object({
|
|
4261
|
+
type: z6.literal("webhook"),
|
|
4262
|
+
url: z6.string(),
|
|
4263
|
+
formatting_prompt: z6.string().min(1).max(1e4).optional()
|
|
4264
|
+
}),
|
|
4105
4265
|
z6.object({
|
|
4106
4266
|
type: z6.literal("slack"),
|
|
4107
4267
|
workspace_id: z6.string(),
|
|
4108
4268
|
channel: z6.string(),
|
|
4109
|
-
message_template: z6.string().optional()
|
|
4269
|
+
message_template: z6.string().optional(),
|
|
4270
|
+
formatting_prompt: z6.string().min(1).max(1e4).optional()
|
|
4110
4271
|
})
|
|
4111
4272
|
])
|
|
4112
4273
|
}),
|
|
@@ -4157,21 +4318,38 @@ var ProjectAutomation = z6.object({
|
|
|
4157
4318
|
}),
|
|
4158
4319
|
z6.object({
|
|
4159
4320
|
event_type: z6.literal("environment_update"),
|
|
4321
|
+
status: AutomationStatus.optional(),
|
|
4160
4322
|
environment_filter: z6.array(z6.string()).optional(),
|
|
4161
4323
|
action: z6.union([
|
|
4162
|
-
z6.object({
|
|
4324
|
+
z6.object({
|
|
4325
|
+
type: z6.literal("webhook"),
|
|
4326
|
+
url: z6.string(),
|
|
4327
|
+
formatting_prompt: z6.string().min(1).max(1e4).optional()
|
|
4328
|
+
}),
|
|
4163
4329
|
z6.object({
|
|
4164
4330
|
type: z6.literal("slack"),
|
|
4165
4331
|
workspace_id: z6.string(),
|
|
4166
4332
|
channel: z6.string(),
|
|
4167
|
-
message_template: z6.string().optional()
|
|
4333
|
+
message_template: z6.string().optional(),
|
|
4334
|
+
formatting_prompt: z6.string().min(1).max(1e4).optional()
|
|
4168
4335
|
})
|
|
4169
4336
|
])
|
|
4170
4337
|
}),
|
|
4338
|
+
WindowedAutomationConfig,
|
|
4171
4339
|
TopicAutomationConfig,
|
|
4172
4340
|
TopicDigestAutomationConfig
|
|
4173
4341
|
])
|
|
4174
4342
|
});
|
|
4343
|
+
var ProjectGroup = z6.object({
|
|
4344
|
+
id: z6.string().uuid(),
|
|
4345
|
+
org_id: z6.string().uuid(),
|
|
4346
|
+
user_id: z6.union([z6.string(), z6.null()]).optional(),
|
|
4347
|
+
created: z6.union([z6.string(), z6.null()]).optional(),
|
|
4348
|
+
name: z6.string(),
|
|
4349
|
+
description: z6.union([z6.string(), z6.null()]).optional(),
|
|
4350
|
+
deleted_at: z6.union([z6.string(), z6.null()]).optional(),
|
|
4351
|
+
member_projects: z6.array(z6.string().uuid()).max(1e4)
|
|
4352
|
+
});
|
|
4175
4353
|
var ProjectLogsEvent = z6.object({
|
|
4176
4354
|
id: z6.string(),
|
|
4177
4355
|
_xact_id: z6.string(),
|
|
@@ -4389,7 +4567,8 @@ var RunEval = z6.object({
|
|
|
4389
4567
|
dataset_environment: z6.union([z6.string(), z6.null()]).optional(),
|
|
4390
4568
|
_internal_btql: z6.union([z6.object({}).partial().passthrough(), z6.null()]).optional()
|
|
4391
4569
|
}),
|
|
4392
|
-
z6.object({ data: z6.array(z6.unknown()) })
|
|
4570
|
+
z6.object({ data: z6.array(z6.unknown()) }),
|
|
4571
|
+
z6.object({ experiment_name: z6.string() })
|
|
4393
4572
|
]),
|
|
4394
4573
|
name: z6.string().optional(),
|
|
4395
4574
|
parameters: z6.object({}).partial().passthrough().optional(),
|
|
@@ -4594,7 +4773,9 @@ var View = z6.object({
|
|
|
4594
4773
|
"for_review_datasets"
|
|
4595
4774
|
]),
|
|
4596
4775
|
name: z6.string(),
|
|
4776
|
+
description: z6.union([z6.string(), z6.null()]).optional(),
|
|
4597
4777
|
created: z6.union([z6.string(), z6.null()]).optional(),
|
|
4778
|
+
updated_at: z6.union([z6.string(), z6.null()]).optional(),
|
|
4598
4779
|
view_data: ViewData.optional(),
|
|
4599
4780
|
options: ViewOptions.optional(),
|
|
4600
4781
|
user_id: z6.union([z6.string(), z6.null()]).optional(),
|
|
@@ -5121,10 +5302,10 @@ var DiskCache = class {
|
|
|
5121
5302
|
return;
|
|
5122
5303
|
}
|
|
5123
5304
|
const stats = await Promise.all(
|
|
5124
|
-
paths.map(async (
|
|
5125
|
-
const stat2 = await isomorph_default.stat(
|
|
5305
|
+
paths.map(async (path3) => {
|
|
5306
|
+
const stat2 = await isomorph_default.stat(path3);
|
|
5126
5307
|
return {
|
|
5127
|
-
path:
|
|
5308
|
+
path: path3,
|
|
5128
5309
|
mtime: stat2.mtime.getTime()
|
|
5129
5310
|
};
|
|
5130
5311
|
})
|
|
@@ -5721,12 +5902,13 @@ var INSTRUMENTATION_NAMES = {
|
|
|
5721
5902
|
OPENROUTER: "openrouter",
|
|
5722
5903
|
OPENROUTER_AGENT: "openrouter-agent",
|
|
5723
5904
|
PI_CODING_AGENT: "pi-coding-agent",
|
|
5724
|
-
STRANDS_AGENT_SDK: "strands-agent-sdk"
|
|
5905
|
+
STRANDS_AGENT_SDK: "strands-agent-sdk",
|
|
5906
|
+
VOYAGEAI: "voyageai"
|
|
5725
5907
|
};
|
|
5726
5908
|
var INTERNAL_SPAN_INSTRUMENTATION_NAME = /* @__PURE__ */ Symbol.for(
|
|
5727
5909
|
"braintrust.spanInstrumentationName"
|
|
5728
5910
|
);
|
|
5729
|
-
var SDK_VERSION = true ? "3.
|
|
5911
|
+
var SDK_VERSION = true ? "3.28.0" : "0.0.0";
|
|
5730
5912
|
function withSpanInstrumentationName(args, instrumentationName) {
|
|
5731
5913
|
return {
|
|
5732
5914
|
...args,
|
|
@@ -5921,6 +6103,9 @@ function applyMaskingToField(maskingFunction, data, fieldName) {
|
|
|
5921
6103
|
var INITIAL_SPAN_WRITE_AS_MERGE = /* @__PURE__ */ Symbol(
|
|
5922
6104
|
"braintrust.initial-span-write-as-merge"
|
|
5923
6105
|
);
|
|
6106
|
+
var RESUME_SPAN_WITHOUT_INITIAL_WRITE = /* @__PURE__ */ Symbol(
|
|
6107
|
+
"braintrust.resume-span-without-initial-write"
|
|
6108
|
+
);
|
|
5924
6109
|
var INTERNAL_SPAN_CONTEXT = /* @__PURE__ */ Symbol("braintrust.internal-span-context");
|
|
5925
6110
|
var BRAINTRUST_CURRENT_SPAN_STORE = /* @__PURE__ */ Symbol.for(
|
|
5926
6111
|
"braintrust.currentSpanStore"
|
|
@@ -6486,9 +6671,9 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
6486
6671
|
this.headers["Authorization"] = `Bearer ${this.token}`;
|
|
6487
6672
|
}
|
|
6488
6673
|
}
|
|
6489
|
-
async get(
|
|
6674
|
+
async get(path3, params = void 0, config) {
|
|
6490
6675
|
const { headers, ...rest } = config || {};
|
|
6491
|
-
const url = new URL(_urljoin(this.base_url,
|
|
6676
|
+
const url = new URL(_urljoin(this.base_url, path3));
|
|
6492
6677
|
url.search = new URLSearchParams(
|
|
6493
6678
|
params ? Object.entries(params).filter(([_, v]) => v !== void 0).flatMap(
|
|
6494
6679
|
([k, v]) => v !== void 0 ? typeof v === "string" ? [[k, v]] : v.map((x) => [k, x]) : []
|
|
@@ -6509,7 +6694,7 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
6509
6694
|
})
|
|
6510
6695
|
);
|
|
6511
6696
|
}
|
|
6512
|
-
async post(
|
|
6697
|
+
async post(path3, params, config, retries = 0) {
|
|
6513
6698
|
const { headers, ...rest } = config || {};
|
|
6514
6699
|
const this_fetch = this.fetch;
|
|
6515
6700
|
const this_base_url = this.base_url;
|
|
@@ -6518,7 +6703,7 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
6518
6703
|
for (let i = 0; i < tries; i++) {
|
|
6519
6704
|
try {
|
|
6520
6705
|
return await checkResponse(
|
|
6521
|
-
await this_fetch(_urljoin(this_base_url,
|
|
6706
|
+
await this_fetch(_urljoin(this_base_url, path3), {
|
|
6522
6707
|
method: "POST",
|
|
6523
6708
|
headers: {
|
|
6524
6709
|
Accept: "application/json",
|
|
@@ -6539,7 +6724,7 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
6539
6724
|
throw error;
|
|
6540
6725
|
}
|
|
6541
6726
|
debugLogger.debug(
|
|
6542
|
-
`Retrying API request ${
|
|
6727
|
+
`Retrying API request ${path3} after ${formatHTTPError(error)}`
|
|
6543
6728
|
);
|
|
6544
6729
|
const sleepTimeMs = HTTP_RETRY_BASE_SLEEP_TIME_S * 1e3 * 2 ** i + Math.random() * HTTP_RETRY_JITTER_MS;
|
|
6545
6730
|
debugLogger.info(
|
|
@@ -9304,7 +9489,7 @@ var ObjectFetcher = class {
|
|
|
9304
9489
|
const objectId = await this.id;
|
|
9305
9490
|
const batchLimit = batchSize ?? DEFAULT_FETCH_BATCH_SIZE;
|
|
9306
9491
|
const internalLimit = getInternalBtqlLimit(this._internal_btql);
|
|
9307
|
-
|
|
9492
|
+
let remainingLimit = internalLimit;
|
|
9308
9493
|
const internalBtqlWithoutReservedQueryKeys = Object.fromEntries(
|
|
9309
9494
|
Object.entries(this._internal_btql ?? {}).filter(
|
|
9310
9495
|
([key]) => key !== "cursor" && key !== "limit" && key !== "select" && key !== "from"
|
|
@@ -9313,6 +9498,10 @@ var ObjectFetcher = class {
|
|
|
9313
9498
|
let cursor = void 0;
|
|
9314
9499
|
let iterations = 0;
|
|
9315
9500
|
while (true) {
|
|
9501
|
+
if (remainingLimit !== void 0 && remainingLimit <= 0) {
|
|
9502
|
+
return;
|
|
9503
|
+
}
|
|
9504
|
+
const limit = remainingLimit === void 0 ? batchLimit : Math.min(batchLimit, remainingLimit);
|
|
9316
9505
|
const resp = await state.apiConn().post(
|
|
9317
9506
|
`btql`,
|
|
9318
9507
|
{
|
|
@@ -9352,7 +9541,14 @@ var ObjectFetcher = class {
|
|
|
9352
9541
|
const respJson = await resp.json();
|
|
9353
9542
|
const mutate = this.mutateRecord;
|
|
9354
9543
|
for (const record of respJson.data ?? []) {
|
|
9355
|
-
|
|
9544
|
+
if (remainingLimit !== void 0 && remainingLimit <= 0) {
|
|
9545
|
+
return;
|
|
9546
|
+
}
|
|
9547
|
+
const mutatedRecord = mutate ? mutate(record) : record;
|
|
9548
|
+
if (remainingLimit !== void 0) {
|
|
9549
|
+
remainingLimit--;
|
|
9550
|
+
}
|
|
9551
|
+
yield mutatedRecord;
|
|
9356
9552
|
}
|
|
9357
9553
|
if (!respJson.cursor) {
|
|
9358
9554
|
break;
|
|
@@ -9872,7 +10068,9 @@ var SpanImpl = class _SpanImpl {
|
|
|
9872
10068
|
this._rootSpanId = resolvedIds.rootSpanId;
|
|
9873
10069
|
this._spanParents = resolvedIds.spanParents;
|
|
9874
10070
|
this.isMerge = args[INITIAL_SPAN_WRITE_AS_MERGE] === true;
|
|
9875
|
-
|
|
10071
|
+
if (!args[RESUME_SPAN_WITHOUT_INITIAL_WRITE]) {
|
|
10072
|
+
this.logInternal({ event, internalData });
|
|
10073
|
+
}
|
|
9876
10074
|
this.isMerge = true;
|
|
9877
10075
|
}
|
|
9878
10076
|
getParentInfo() {
|
|
@@ -12430,6 +12628,22 @@ function processInputAttachments(input) {
|
|
|
12430
12628
|
};
|
|
12431
12629
|
}
|
|
12432
12630
|
}
|
|
12631
|
+
const voyageBase64Key = node.type === "image_base64" ? Object.hasOwn(node, "imageBase64") ? "imageBase64" : "image_base64" : node.type === "video_base64" ? Object.hasOwn(node, "videoBase64") ? "videoBase64" : "video_base64" : void 0;
|
|
12632
|
+
const voyageBase64Value = voyageBase64Key ? node[voyageBase64Key] : void 0;
|
|
12633
|
+
if (voyageBase64Key && typeof voyageBase64Value === "string" && voyageBase64Value.startsWith("data:")) {
|
|
12634
|
+
const mediaType = inferMediaTypeFromDataUrl(
|
|
12635
|
+
voyageBase64Value,
|
|
12636
|
+
node.type === "video_base64" ? "video/mp4" : "image/png"
|
|
12637
|
+
);
|
|
12638
|
+
const filename = `${node.type === "video_base64" ? "video" : "image"}.${getExtensionFromMediaType(mediaType)}`;
|
|
12639
|
+
const attachment = toAttachment(voyageBase64Value, mediaType, filename);
|
|
12640
|
+
if (attachment) {
|
|
12641
|
+
return {
|
|
12642
|
+
...node,
|
|
12643
|
+
[voyageBase64Key]: attachment
|
|
12644
|
+
};
|
|
12645
|
+
}
|
|
12646
|
+
}
|
|
12433
12647
|
if (node.type === "file" && node.file && typeof node.file === "object" && typeof node.file.file_data === "string" && node.file.file_data.startsWith("data:")) {
|
|
12434
12648
|
const mediaType = inferMediaTypeFromDataUrl(
|
|
12435
12649
|
node.file.file_data,
|
|
@@ -13777,12 +13991,30 @@ function logInstrumentationError(context, error) {
|
|
|
13777
13991
|
|
|
13778
13992
|
// src/wrappers/anthropic-tokens-util.ts
|
|
13779
13993
|
function finalizeAnthropicTokens(metrics) {
|
|
13780
|
-
const
|
|
13781
|
-
|
|
13994
|
+
const hasSplitCacheCreationTokens = metrics.prompt_cache_creation_5m_tokens !== void 0 || metrics.prompt_cache_creation_1h_tokens !== void 0;
|
|
13995
|
+
const splitCacheCreationTokens = (metrics.prompt_cache_creation_5m_tokens || 0) + (metrics.prompt_cache_creation_1h_tokens || 0);
|
|
13996
|
+
const aggregateCacheCreationTokens = metrics.prompt_cache_creation_tokens || 0;
|
|
13997
|
+
const effectiveCacheCreationTokens = Math.max(
|
|
13998
|
+
aggregateCacheCreationTokens,
|
|
13999
|
+
splitCacheCreationTokens
|
|
14000
|
+
);
|
|
14001
|
+
const prompt_tokens = (metrics.prompt_tokens || 0) + (metrics.prompt_cached_tokens || 0) + effectiveCacheCreationTokens;
|
|
14002
|
+
const finalized = {
|
|
13782
14003
|
...metrics,
|
|
13783
14004
|
prompt_tokens,
|
|
13784
14005
|
tokens: prompt_tokens + (metrics.completion_tokens || 0)
|
|
13785
14006
|
};
|
|
14007
|
+
if (hasSplitCacheCreationTokens && splitCacheCreationTokens >= aggregateCacheCreationTokens) {
|
|
14008
|
+
delete finalized.prompt_cache_creation_tokens;
|
|
14009
|
+
}
|
|
14010
|
+
return finalized;
|
|
14011
|
+
}
|
|
14012
|
+
function toNumericMetrics(metrics) {
|
|
14013
|
+
return Object.fromEntries(
|
|
14014
|
+
Object.entries(metrics).filter(
|
|
14015
|
+
(entry) => entry[1] !== void 0
|
|
14016
|
+
)
|
|
14017
|
+
);
|
|
13786
14018
|
}
|
|
13787
14019
|
function extractAnthropicCacheTokens(cacheReadTokens = 0, cacheCreationTokens = 0) {
|
|
13788
14020
|
const cacheTokens = {};
|
|
@@ -14375,6 +14607,18 @@ function parseMetricsFromUsage2(usage) {
|
|
|
14375
14607
|
saveIfExistsTo("output_tokens", "completion_tokens");
|
|
14376
14608
|
saveIfExistsTo("cache_read_input_tokens", "prompt_cached_tokens");
|
|
14377
14609
|
saveIfExistsTo("cache_creation_input_tokens", "prompt_cache_creation_tokens");
|
|
14610
|
+
if (isObject(usage.cache_creation)) {
|
|
14611
|
+
const cacheCreation = usage.cache_creation;
|
|
14612
|
+
for (const [source, target] of [
|
|
14613
|
+
["ephemeral_5m_input_tokens", "prompt_cache_creation_5m_tokens"],
|
|
14614
|
+
["ephemeral_1h_input_tokens", "prompt_cache_creation_1h_tokens"]
|
|
14615
|
+
]) {
|
|
14616
|
+
const value = cacheCreation[source];
|
|
14617
|
+
if (typeof value === "number") {
|
|
14618
|
+
metrics[target] = value;
|
|
14619
|
+
}
|
|
14620
|
+
}
|
|
14621
|
+
}
|
|
14378
14622
|
if (isObject(usage.server_tool_use)) {
|
|
14379
14623
|
for (const [name, value] of Object.entries(usage.server_tool_use)) {
|
|
14380
14624
|
if (typeof value === "number") {
|
|
@@ -16966,7 +17210,7 @@ function resolveDenyOutputPaths(event, defaultDenyOutputPaths) {
|
|
|
16966
17210
|
return defaultDenyOutputPaths;
|
|
16967
17211
|
}
|
|
16968
17212
|
const runtimeDenyOutputPaths = firstArgument2[RUNTIME_DENY_OUTPUT_PATHS];
|
|
16969
|
-
if (Array.isArray(runtimeDenyOutputPaths) && runtimeDenyOutputPaths.every((
|
|
17213
|
+
if (Array.isArray(runtimeDenyOutputPaths) && runtimeDenyOutputPaths.every((path3) => typeof path3 === "string")) {
|
|
16970
17214
|
return runtimeDenyOutputPaths;
|
|
16971
17215
|
}
|
|
16972
17216
|
return defaultDenyOutputPaths;
|
|
@@ -18728,11 +18972,11 @@ function processAISDKOutput(output, denyOutputPaths) {
|
|
|
18728
18972
|
if (!output) return output;
|
|
18729
18973
|
const merged = extractSerializableOutputFields(output);
|
|
18730
18974
|
const deleteOutputPaths = denyOutputPaths.filter(
|
|
18731
|
-
(
|
|
18975
|
+
(path3) => path3.toLowerCase().endsWith("headers")
|
|
18732
18976
|
);
|
|
18733
18977
|
const sanitized = omit(merged, denyOutputPaths, deleteOutputPaths);
|
|
18734
|
-
for (const
|
|
18735
|
-
const stack = [{ obj: sanitized, keys: parsePath(
|
|
18978
|
+
for (const path3 of TRANSPORT_PAYLOAD_ROOT_PATHS) {
|
|
18979
|
+
const stack = [{ obj: sanitized, keys: parsePath(path3) }];
|
|
18736
18980
|
while (stack.length > 0) {
|
|
18737
18981
|
const entry = stack.pop();
|
|
18738
18982
|
if (!entry || entry.keys.length === 0) {
|
|
@@ -19171,11 +19415,11 @@ function firstNumber2(...values) {
|
|
|
19171
19415
|
function deepCopy(obj) {
|
|
19172
19416
|
return JSON.parse(JSON.stringify(obj));
|
|
19173
19417
|
}
|
|
19174
|
-
function parsePath(
|
|
19418
|
+
function parsePath(path3) {
|
|
19175
19419
|
const keys = [];
|
|
19176
19420
|
let current = "";
|
|
19177
|
-
for (let i = 0; i <
|
|
19178
|
-
const char =
|
|
19421
|
+
for (let i = 0; i < path3.length; i++) {
|
|
19422
|
+
const char = path3[i];
|
|
19179
19423
|
if (char === ".") {
|
|
19180
19424
|
if (current) {
|
|
19181
19425
|
keys.push(current);
|
|
@@ -19188,8 +19432,8 @@ function parsePath(path2) {
|
|
|
19188
19432
|
}
|
|
19189
19433
|
let bracketContent = "";
|
|
19190
19434
|
i++;
|
|
19191
|
-
while (i <
|
|
19192
|
-
bracketContent +=
|
|
19435
|
+
while (i < path3.length && path3[i] !== "]") {
|
|
19436
|
+
bracketContent += path3[i];
|
|
19193
19437
|
i++;
|
|
19194
19438
|
}
|
|
19195
19439
|
if (bracketContent === "") {
|
|
@@ -19244,9 +19488,9 @@ function omitAtPath(obj, keys, deleteLeaf = false) {
|
|
|
19244
19488
|
function omit(obj, paths, deletePaths = []) {
|
|
19245
19489
|
const result = deepCopy(obj);
|
|
19246
19490
|
const deletePathSet = new Set(deletePaths);
|
|
19247
|
-
for (const
|
|
19248
|
-
const keys = parsePath(
|
|
19249
|
-
omitAtPath(result, keys, deletePathSet.has(
|
|
19491
|
+
for (const path3 of paths) {
|
|
19492
|
+
const keys = parsePath(path3);
|
|
19493
|
+
omitAtPath(result, keys, deletePathSet.has(path3));
|
|
19250
19494
|
}
|
|
19251
19495
|
return result;
|
|
19252
19496
|
}
|
|
@@ -19673,37 +19917,97 @@ function seedTaskToolUseIdMapping(taskIdToToolUseId, message) {
|
|
|
19673
19917
|
taskIdToToolUseId.set(message.task_id, message.tool_use_id);
|
|
19674
19918
|
}
|
|
19675
19919
|
}
|
|
19676
|
-
function
|
|
19677
|
-
|
|
19678
|
-
|
|
19679
|
-
|
|
19680
|
-
usage = message.message?.usage;
|
|
19681
|
-
} else if (message.type === "result") {
|
|
19682
|
-
usage = message.usage;
|
|
19683
|
-
}
|
|
19920
|
+
function tokenCount(value) {
|
|
19921
|
+
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value) && value >= 0 ? value : void 0;
|
|
19922
|
+
}
|
|
19923
|
+
function copyUsage(usage) {
|
|
19684
19924
|
if (!usage || typeof usage !== "object") {
|
|
19685
|
-
return
|
|
19925
|
+
return void 0;
|
|
19926
|
+
}
|
|
19927
|
+
const copy = {};
|
|
19928
|
+
for (const key of [
|
|
19929
|
+
"input_tokens",
|
|
19930
|
+
"output_tokens",
|
|
19931
|
+
"cache_read_input_tokens",
|
|
19932
|
+
"cache_creation_input_tokens"
|
|
19933
|
+
]) {
|
|
19934
|
+
const value = tokenCount(Reflect.get(usage, key));
|
|
19935
|
+
if (value !== void 0) {
|
|
19936
|
+
copy[key] = value;
|
|
19937
|
+
}
|
|
19938
|
+
}
|
|
19939
|
+
const cacheCreation = Reflect.get(usage, "cache_creation");
|
|
19940
|
+
if (cacheCreation && typeof cacheCreation === "object") {
|
|
19941
|
+
const cacheCreationCopy = {};
|
|
19942
|
+
for (const key of [
|
|
19943
|
+
"ephemeral_5m_input_tokens",
|
|
19944
|
+
"ephemeral_1h_input_tokens"
|
|
19945
|
+
]) {
|
|
19946
|
+
const value = tokenCount(Reflect.get(cacheCreation, key));
|
|
19947
|
+
if (value !== void 0) {
|
|
19948
|
+
cacheCreationCopy[key] = value;
|
|
19949
|
+
}
|
|
19950
|
+
}
|
|
19951
|
+
if (Object.keys(cacheCreationCopy).length > 0) {
|
|
19952
|
+
copy.cache_creation = cacheCreationCopy;
|
|
19953
|
+
}
|
|
19954
|
+
}
|
|
19955
|
+
return Object.keys(copy).length > 0 ? copy : void 0;
|
|
19956
|
+
}
|
|
19957
|
+
function mergeUsage(base, override) {
|
|
19958
|
+
if (!base || !override) {
|
|
19959
|
+
return override ?? base;
|
|
19960
|
+
}
|
|
19961
|
+
const cacheCreation = base.cache_creation || override.cache_creation ? { ...base.cache_creation, ...override.cache_creation } : void 0;
|
|
19962
|
+
return {
|
|
19963
|
+
...base,
|
|
19964
|
+
...override,
|
|
19965
|
+
...cacheCreation && { cache_creation: cacheCreation }
|
|
19966
|
+
};
|
|
19967
|
+
}
|
|
19968
|
+
function extractUsage(usage, includeOutput) {
|
|
19969
|
+
const metrics = {};
|
|
19970
|
+
if (!usage) {
|
|
19971
|
+
return {};
|
|
19686
19972
|
}
|
|
19687
19973
|
const inputTokens = getNumberProperty(usage, "input_tokens");
|
|
19688
19974
|
if (inputTokens !== void 0) {
|
|
19689
19975
|
metrics.prompt_tokens = inputTokens;
|
|
19690
19976
|
}
|
|
19691
|
-
|
|
19692
|
-
|
|
19693
|
-
|
|
19977
|
+
if (includeOutput) {
|
|
19978
|
+
const outputTokens = getNumberProperty(usage, "output_tokens");
|
|
19979
|
+
if (outputTokens !== void 0) {
|
|
19980
|
+
metrics.completion_tokens = outputTokens;
|
|
19981
|
+
}
|
|
19694
19982
|
}
|
|
19695
19983
|
const cacheReadTokens = getNumberProperty(usage, "cache_read_input_tokens") || 0;
|
|
19696
19984
|
const cacheCreationTokens = getNumberProperty(usage, "cache_creation_input_tokens") || 0;
|
|
19697
|
-
|
|
19698
|
-
|
|
19699
|
-
|
|
19700
|
-
|
|
19701
|
-
|
|
19985
|
+
Object.assign(
|
|
19986
|
+
metrics,
|
|
19987
|
+
extractAnthropicCacheTokens(cacheReadTokens, cacheCreationTokens)
|
|
19988
|
+
);
|
|
19989
|
+
const cacheCreation5mTokens = getNumberProperty(
|
|
19990
|
+
usage.cache_creation,
|
|
19991
|
+
"ephemeral_5m_input_tokens"
|
|
19992
|
+
);
|
|
19993
|
+
const cacheCreation1hTokens = getNumberProperty(
|
|
19994
|
+
usage.cache_creation,
|
|
19995
|
+
"ephemeral_1h_input_tokens"
|
|
19996
|
+
);
|
|
19997
|
+
if (cacheCreation5mTokens !== void 0) {
|
|
19998
|
+
metrics.prompt_cache_creation_5m_tokens = cacheCreation5mTokens;
|
|
19702
19999
|
}
|
|
19703
|
-
if (
|
|
19704
|
-
|
|
20000
|
+
if (cacheCreation1hTokens !== void 0) {
|
|
20001
|
+
metrics.prompt_cache_creation_1h_tokens = cacheCreation1hTokens;
|
|
19705
20002
|
}
|
|
19706
|
-
|
|
20003
|
+
if (Object.keys(metrics).length === 0) {
|
|
20004
|
+
return {};
|
|
20005
|
+
}
|
|
20006
|
+
const finalized = finalizeAnthropicTokens(metrics);
|
|
20007
|
+
if (metrics.completion_tokens === void 0) {
|
|
20008
|
+
delete finalized.tokens;
|
|
20009
|
+
}
|
|
20010
|
+
return toNumericMetrics(finalized);
|
|
19707
20011
|
}
|
|
19708
20012
|
function buildLLMInput(promptMessages, conversationHistory) {
|
|
19709
20013
|
const inputParts = [...promptMessages, ...conversationHistory];
|
|
@@ -19737,16 +20041,16 @@ function buildRootPromptMessages(prompt, capturedPromptMessages) {
|
|
|
19737
20041
|
function formatCapturedMessages(messages) {
|
|
19738
20042
|
return messages.length > 0 ? messages : [];
|
|
19739
20043
|
}
|
|
19740
|
-
async function createLLMSpanForMessages(messages, promptMessages, conversationHistory, options, startTime, parentSpan, existingSpan) {
|
|
20044
|
+
async function createLLMSpanForMessages(messages, promptMessages, conversationHistory, options, startTime, parentSpan, usage, hasFinalOutputUsage, existingSpan) {
|
|
19741
20045
|
if (messages.length === 0) {
|
|
19742
20046
|
return void 0;
|
|
19743
20047
|
}
|
|
19744
20048
|
const lastMessage = messages[messages.length - 1];
|
|
19745
|
-
if (lastMessage.type !== "assistant"
|
|
20049
|
+
if (lastMessage.type !== "assistant") {
|
|
19746
20050
|
return void 0;
|
|
19747
20051
|
}
|
|
19748
|
-
const model = lastMessage.message
|
|
19749
|
-
const
|
|
20052
|
+
const model = lastMessage.message?.model || options.model;
|
|
20053
|
+
const metrics = options.includePartialMessages ? extractUsage(usage, hasFinalOutputUsage) : {};
|
|
19750
20054
|
const input = buildLLMInput(promptMessages, conversationHistory);
|
|
19751
20055
|
const outputs = messages.map(
|
|
19752
20056
|
(m) => m.message?.content && m.message?.role ? { content: m.message.content, role: m.message.role } : void 0
|
|
@@ -19768,8 +20072,8 @@ async function createLLMSpanForMessages(messages, promptMessages, conversationHi
|
|
|
19768
20072
|
);
|
|
19769
20073
|
span.log({
|
|
19770
20074
|
input,
|
|
19771
|
-
metadata: model
|
|
19772
|
-
metrics:
|
|
20075
|
+
metadata: { ...model && { model }, provider: "anthropic" },
|
|
20076
|
+
...Object.keys(metrics).length > 0 ? { metrics } : {},
|
|
19773
20077
|
output: outputs
|
|
19774
20078
|
});
|
|
19775
20079
|
const spanExport = await span.export();
|
|
@@ -19859,11 +20163,17 @@ function prepareLocalToolHandlersInMcpServers(mcpServers) {
|
|
|
19859
20163
|
}
|
|
19860
20164
|
return { hasLocalToolHandlers, localToolHookNames };
|
|
19861
20165
|
}
|
|
19862
|
-
function createToolTracingHooks(resolveParentSpan, activeToolSpans, mcpServers, localToolHookNames, skipLocalToolHooks, subAgentDetailsByToolUseId, subAgentSpans, endedSubAgentSpans) {
|
|
20166
|
+
function createToolTracingHooks(resolveParentSpan, taskIdToToolUseId, toolUseToParent, activeToolSpans, mcpServers, localToolHookNames, skipLocalToolHooks, subAgentDetailsByToolUseId, subAgentSpans, endedSubAgentSpans) {
|
|
19863
20167
|
const preToolUse = async (input, toolUseID) => {
|
|
19864
20168
|
if (input.hook_event_name !== "PreToolUse" || !toolUseID) {
|
|
19865
20169
|
return {};
|
|
19866
20170
|
}
|
|
20171
|
+
if (!toolUseToParent.has(toolUseID) && input.agent_id) {
|
|
20172
|
+
const parentToolUseId = taskIdToToolUseId.get(input.agent_id);
|
|
20173
|
+
if (parentToolUseId) {
|
|
20174
|
+
toolUseToParent.set(toolUseID, parentToolUseId);
|
|
20175
|
+
}
|
|
20176
|
+
}
|
|
19867
20177
|
if (skipLocalToolHooks && (isLocalToolUse(input.tool_name, mcpServers) || localToolHookNames.has(input.tool_name))) {
|
|
19868
20178
|
return {};
|
|
19869
20179
|
}
|
|
@@ -20059,9 +20369,6 @@ function createToolTracingHooks(resolveParentSpan, activeToolSpans, mcpServers,
|
|
|
20059
20369
|
}
|
|
20060
20370
|
const metadata = {
|
|
20061
20371
|
...subAgentDetailsToMetadata(details),
|
|
20062
|
-
...input.agent_transcript_path && {
|
|
20063
|
-
"claude_agent_sdk.agent_transcript_path": input.agent_transcript_path
|
|
20064
|
-
},
|
|
20065
20372
|
"claude_agent_sdk.stop_hook_active": input.stop_hook_active
|
|
20066
20373
|
};
|
|
20067
20374
|
try {
|
|
@@ -20083,7 +20390,7 @@ function createToolTracingHooks(resolveParentSpan, activeToolSpans, mcpServers,
|
|
|
20083
20390
|
subagentStop
|
|
20084
20391
|
};
|
|
20085
20392
|
}
|
|
20086
|
-
function injectTracingHooks(options, resolveParentSpan, activeToolSpans, localToolHookNames, skipLocalToolHooks, subAgentDetailsByToolUseId, subAgentSpans, endedSubAgentSpans) {
|
|
20393
|
+
function injectTracingHooks(options, resolveParentSpan, taskIdToToolUseId, toolUseToParent, activeToolSpans, localToolHookNames, skipLocalToolHooks, subAgentDetailsByToolUseId, subAgentSpans, endedSubAgentSpans) {
|
|
20087
20394
|
const {
|
|
20088
20395
|
preToolUse,
|
|
20089
20396
|
postToolUse,
|
|
@@ -20092,6 +20399,8 @@ function injectTracingHooks(options, resolveParentSpan, activeToolSpans, localTo
|
|
|
20092
20399
|
subagentStop
|
|
20093
20400
|
} = createToolTracingHooks(
|
|
20094
20401
|
resolveParentSpan,
|
|
20402
|
+
taskIdToToolUseId,
|
|
20403
|
+
toolUseToParent,
|
|
20095
20404
|
activeToolSpans,
|
|
20096
20405
|
options.mcpServers,
|
|
20097
20406
|
localToolHookNames,
|
|
@@ -20173,6 +20482,13 @@ async function finalizeCurrentMessageGroup(state) {
|
|
|
20173
20482
|
}
|
|
20174
20483
|
}
|
|
20175
20484
|
const existingLlmSpan = state.activeLlmSpansByParentToolUse.get(parentKey);
|
|
20485
|
+
const lastMessage = state.currentMessages[state.currentMessages.length - 1];
|
|
20486
|
+
const messageId = lastMessage?.message?.id;
|
|
20487
|
+
const usage = state.options.includePartialMessages ? mergeUsage(
|
|
20488
|
+
copyUsage(lastMessage?.message?.usage),
|
|
20489
|
+
messageId ? state.usageByMessageId.get(messageId) : void 0
|
|
20490
|
+
) : void 0;
|
|
20491
|
+
const hasFinalOutputUsage = messageId !== void 0 && state.finalOutputUsageMessageIds.has(messageId);
|
|
20176
20492
|
const llmSpanResult = await createLLMSpanForMessages(
|
|
20177
20493
|
state.currentMessages,
|
|
20178
20494
|
promptMessages,
|
|
@@ -20180,6 +20496,8 @@ async function finalizeCurrentMessageGroup(state) {
|
|
|
20180
20496
|
state.options,
|
|
20181
20497
|
state.currentMessageStartTime,
|
|
20182
20498
|
parentSpan,
|
|
20499
|
+
usage,
|
|
20500
|
+
hasFinalOutputUsage,
|
|
20183
20501
|
existingLlmSpan
|
|
20184
20502
|
);
|
|
20185
20503
|
if (llmSpanResult) {
|
|
@@ -20197,9 +20515,17 @@ async function finalizeCurrentMessageGroup(state) {
|
|
|
20197
20515
|
}
|
|
20198
20516
|
}
|
|
20199
20517
|
state.activeLlmSpansByParentToolUse.delete(parentKey);
|
|
20200
|
-
|
|
20201
|
-
|
|
20202
|
-
state.
|
|
20518
|
+
if (messageId) {
|
|
20519
|
+
state.usageByMessageId.delete(messageId);
|
|
20520
|
+
state.finalOutputUsageMessageIds.delete(messageId);
|
|
20521
|
+
for (const [
|
|
20522
|
+
parent,
|
|
20523
|
+
activeMessageId
|
|
20524
|
+
] of state.activePartialMessageIdByParentKey) {
|
|
20525
|
+
if (activeMessageId === messageId) {
|
|
20526
|
+
state.activePartialMessageIdByParentKey.delete(parent);
|
|
20527
|
+
}
|
|
20528
|
+
}
|
|
20203
20529
|
}
|
|
20204
20530
|
state.currentMessages.length = 0;
|
|
20205
20531
|
}
|
|
@@ -20290,6 +20616,10 @@ async function ensureActiveLlmSpanForParentToolUse(rootSpan, activeLlmSpansByPar
|
|
|
20290
20616
|
);
|
|
20291
20617
|
llmParentSpan = await subAgentSpan.export();
|
|
20292
20618
|
}
|
|
20619
|
+
const racedLlmSpan = activeLlmSpansByParentToolUse.get(parentKey);
|
|
20620
|
+
if (racedLlmSpan) {
|
|
20621
|
+
return racedLlmSpan;
|
|
20622
|
+
}
|
|
20293
20623
|
const llmSpan = startSpan(
|
|
20294
20624
|
withSpanInstrumentationName(
|
|
20295
20625
|
{
|
|
@@ -20409,7 +20739,49 @@ async function maybeHandleTaskLifecycleMessage(state, message) {
|
|
|
20409
20739
|
}
|
|
20410
20740
|
return true;
|
|
20411
20741
|
}
|
|
20742
|
+
function handlePartialUsageMessage(state, message) {
|
|
20743
|
+
if (message.type !== "stream_event") {
|
|
20744
|
+
return false;
|
|
20745
|
+
}
|
|
20746
|
+
const event = message.event;
|
|
20747
|
+
if (!event || typeof event !== "object") {
|
|
20748
|
+
return true;
|
|
20749
|
+
}
|
|
20750
|
+
const parentKey = llmParentKey(message.parent_tool_use_id ?? null);
|
|
20751
|
+
if (event.type === "message_start") {
|
|
20752
|
+
const messageId2 = event.message?.id;
|
|
20753
|
+
const usage = copyUsage(event.message?.usage);
|
|
20754
|
+
if (messageId2) {
|
|
20755
|
+
state.activePartialMessageIdByParentKey.set(parentKey, messageId2);
|
|
20756
|
+
if (usage) {
|
|
20757
|
+
state.usageByMessageId.set(messageId2, usage);
|
|
20758
|
+
}
|
|
20759
|
+
}
|
|
20760
|
+
return true;
|
|
20761
|
+
}
|
|
20762
|
+
const messageId = state.activePartialMessageIdByParentKey.get(parentKey);
|
|
20763
|
+
if (!messageId) {
|
|
20764
|
+
return true;
|
|
20765
|
+
}
|
|
20766
|
+
if (event.type === "message_delta") {
|
|
20767
|
+
const update = copyUsage(event.usage);
|
|
20768
|
+
if (update) {
|
|
20769
|
+
const usage = state.usageByMessageId.get(messageId) ?? {};
|
|
20770
|
+
Object.assign(usage, update);
|
|
20771
|
+
state.usageByMessageId.set(messageId, usage);
|
|
20772
|
+
if (update.output_tokens !== void 0) {
|
|
20773
|
+
state.finalOutputUsageMessageIds.add(messageId);
|
|
20774
|
+
}
|
|
20775
|
+
}
|
|
20776
|
+
} else if (event.type === "message_stop") {
|
|
20777
|
+
state.activePartialMessageIdByParentKey.delete(parentKey);
|
|
20778
|
+
}
|
|
20779
|
+
return true;
|
|
20780
|
+
}
|
|
20412
20781
|
async function handleStreamMessage(state, message) {
|
|
20782
|
+
if (handlePartialUsageMessage(state, message)) {
|
|
20783
|
+
return;
|
|
20784
|
+
}
|
|
20413
20785
|
maybeTrackToolUseContext(state, message);
|
|
20414
20786
|
if (await maybeHandleTaskLifecycleMessage(state, message)) {
|
|
20415
20787
|
return;
|
|
@@ -20456,36 +20828,9 @@ async function handleStreamMessage(state, message) {
|
|
|
20456
20828
|
);
|
|
20457
20829
|
state.currentMessages.push(message);
|
|
20458
20830
|
}
|
|
20459
|
-
if (message.type !== "result"
|
|
20831
|
+
if (message.type !== "result") {
|
|
20460
20832
|
return;
|
|
20461
20833
|
}
|
|
20462
|
-
const finalUsageMetrics = extractUsageFromMessage(message);
|
|
20463
|
-
if (state.currentMessages.length > 0 && finalUsageMetrics.completion_tokens !== void 0) {
|
|
20464
|
-
const lastMessage = state.currentMessages[state.currentMessages.length - 1];
|
|
20465
|
-
if (lastMessage?.message?.usage) {
|
|
20466
|
-
const adjustedTokens = finalUsageMetrics.completion_tokens - state.accumulatedOutputTokens;
|
|
20467
|
-
if (adjustedTokens >= 0) {
|
|
20468
|
-
lastMessage.message.usage.output_tokens = adjustedTokens;
|
|
20469
|
-
}
|
|
20470
|
-
const resultUsage = message.usage;
|
|
20471
|
-
if (resultUsage && typeof resultUsage === "object") {
|
|
20472
|
-
const cacheReadTokens = getNumberProperty(
|
|
20473
|
-
resultUsage,
|
|
20474
|
-
"cache_read_input_tokens"
|
|
20475
|
-
);
|
|
20476
|
-
if (cacheReadTokens !== void 0) {
|
|
20477
|
-
lastMessage.message.usage.cache_read_input_tokens = cacheReadTokens;
|
|
20478
|
-
}
|
|
20479
|
-
const cacheCreationTokens = getNumberProperty(
|
|
20480
|
-
resultUsage,
|
|
20481
|
-
"cache_creation_input_tokens"
|
|
20482
|
-
);
|
|
20483
|
-
if (cacheCreationTokens !== void 0) {
|
|
20484
|
-
lastMessage.message.usage.cache_creation_input_tokens = cacheCreationTokens;
|
|
20485
|
-
}
|
|
20486
|
-
}
|
|
20487
|
-
}
|
|
20488
|
-
}
|
|
20489
20834
|
const metadata = {};
|
|
20490
20835
|
if (message.num_turns !== void 0) {
|
|
20491
20836
|
metadata.num_turns = message.num_turns;
|
|
@@ -20493,8 +20838,12 @@ async function handleStreamMessage(state, message) {
|
|
|
20493
20838
|
if (message.session_id !== void 0) {
|
|
20494
20839
|
metadata.session_id = message.session_id;
|
|
20495
20840
|
}
|
|
20496
|
-
|
|
20497
|
-
|
|
20841
|
+
const metrics = state.options.includePartialMessages ? {} : extractUsage(copyUsage(message.usage), true);
|
|
20842
|
+
if (Object.keys(metadata).length > 0 || Object.keys(metrics).length > 0) {
|
|
20843
|
+
state.span.log({
|
|
20844
|
+
...Object.keys(metadata).length > 0 ? { metadata } : {},
|
|
20845
|
+
...Object.keys(metrics).length > 0 ? { metrics } : {}
|
|
20846
|
+
});
|
|
20498
20847
|
}
|
|
20499
20848
|
}
|
|
20500
20849
|
async function finalizeQuerySpan(state) {
|
|
@@ -20518,6 +20867,9 @@ async function finalizeQuerySpan(state) {
|
|
|
20518
20867
|
llmSpan.end();
|
|
20519
20868
|
}
|
|
20520
20869
|
state.activeLlmSpansByParentToolUse.clear();
|
|
20870
|
+
state.activePartialMessageIdByParentKey.clear();
|
|
20871
|
+
state.finalOutputUsageMessageIds.clear();
|
|
20872
|
+
state.usageByMessageId.clear();
|
|
20521
20873
|
for (const toolSpan of state.activeToolSpans.values()) {
|
|
20522
20874
|
toolSpan.end();
|
|
20523
20875
|
}
|
|
@@ -20642,6 +20994,8 @@ var ClaudeAgentSDKPlugin = class extends BasePlugin {
|
|
|
20642
20994
|
const optionsWithHooks = injectTracingHooks(
|
|
20643
20995
|
options,
|
|
20644
20996
|
resolveToolUseParentSpan,
|
|
20997
|
+
taskIdToToolUseId,
|
|
20998
|
+
toolUseToParent,
|
|
20645
20999
|
activeToolSpans,
|
|
20646
21000
|
localToolHookNames,
|
|
20647
21001
|
skipLocalToolHooks,
|
|
@@ -20652,8 +21006,8 @@ var ClaudeAgentSDKPlugin = class extends BasePlugin {
|
|
|
20652
21006
|
params.options = optionsWithHooks;
|
|
20653
21007
|
event.arguments[0] = params;
|
|
20654
21008
|
spans.set(event, {
|
|
20655
|
-
accumulatedOutputTokens: 0,
|
|
20656
21009
|
activeLlmSpansByParentToolUse,
|
|
21010
|
+
activePartialMessageIdByParentKey: /* @__PURE__ */ new Map(),
|
|
20657
21011
|
activeToolSpans,
|
|
20658
21012
|
conversationHistoryByParentKey,
|
|
20659
21013
|
capturedPromptMessages,
|
|
@@ -20661,6 +21015,7 @@ var ClaudeAgentSDKPlugin = class extends BasePlugin {
|
|
|
20661
21015
|
currentMessageStartTime: startTime,
|
|
20662
21016
|
currentMessages: [],
|
|
20663
21017
|
endedSubAgentSpans,
|
|
21018
|
+
finalOutputUsageMessageIds: /* @__PURE__ */ new Set(),
|
|
20664
21019
|
finalResults: [],
|
|
20665
21020
|
options: optionsWithHooks,
|
|
20666
21021
|
originalPrompt,
|
|
@@ -20676,6 +21031,7 @@ var ClaudeAgentSDKPlugin = class extends BasePlugin {
|
|
|
20676
21031
|
latestLlmParentBySubAgentToolUse,
|
|
20677
21032
|
latestRootLlmParentRef,
|
|
20678
21033
|
toolUseToParent,
|
|
21034
|
+
usageByMessageId: /* @__PURE__ */ new Map(),
|
|
20679
21035
|
localToolContext
|
|
20680
21036
|
});
|
|
20681
21037
|
},
|
|
@@ -23150,9 +23506,9 @@ function extractEmbedPromptTokenCount(response) {
|
|
|
23150
23506
|
let sawAny = false;
|
|
23151
23507
|
for (const embedding of embeddings) {
|
|
23152
23508
|
const embeddingStats = tryToDict(tryToDict(embedding)?.statistics);
|
|
23153
|
-
const
|
|
23154
|
-
if (typeof
|
|
23155
|
-
total +=
|
|
23509
|
+
const tokenCount2 = embeddingStats?.tokenCount;
|
|
23510
|
+
if (typeof tokenCount2 === "number" && Number.isFinite(tokenCount2)) {
|
|
23511
|
+
total += tokenCount2;
|
|
23156
23512
|
sawAny = true;
|
|
23157
23513
|
}
|
|
23158
23514
|
}
|
|
@@ -30047,7 +30403,7 @@ function getStringProperty2(obj, key) {
|
|
|
30047
30403
|
return typeof value === "string" ? value : void 0;
|
|
30048
30404
|
}
|
|
30049
30405
|
function extractMetricsFromUsage(usage) {
|
|
30050
|
-
const
|
|
30406
|
+
const rawMetrics = {
|
|
30051
30407
|
prompt_tokens: usage.inputTokens,
|
|
30052
30408
|
completion_tokens: usage.outputTokens,
|
|
30053
30409
|
...extractAnthropicCacheTokens(
|
|
@@ -30056,10 +30412,10 @@ function extractMetricsFromUsage(usage) {
|
|
|
30056
30412
|
)
|
|
30057
30413
|
};
|
|
30058
30414
|
if (usage.reasoningTokens !== void 0) {
|
|
30059
|
-
|
|
30060
|
-
|
|
30415
|
+
rawMetrics.completion_reasoning_tokens = usage.reasoningTokens;
|
|
30416
|
+
rawMetrics.reasoning_tokens = usage.reasoningTokens;
|
|
30061
30417
|
}
|
|
30062
|
-
|
|
30418
|
+
const metrics = finalizeAnthropicTokens(rawMetrics);
|
|
30063
30419
|
const metadata = {
|
|
30064
30420
|
model: usage.model
|
|
30065
30421
|
};
|
|
@@ -30998,17 +31354,20 @@ var FlueObserveBridge = class {
|
|
|
30998
31354
|
return;
|
|
30999
31355
|
}
|
|
31000
31356
|
const metadata = {
|
|
31357
|
+
...event.runId ? this.runsById.get(event.runId)?.metadata : {},
|
|
31001
31358
|
...extractEventMetadata(event),
|
|
31002
31359
|
"flue.operation": event.operationKind,
|
|
31003
31360
|
provider: "flue"
|
|
31004
31361
|
};
|
|
31005
31362
|
const parent = this.parentSpanForEvent(event);
|
|
31006
|
-
const
|
|
31363
|
+
const args = {
|
|
31007
31364
|
name: `flue.${event.operationKind}`,
|
|
31008
31365
|
spanAttributes: { type: "task" /* TASK */ },
|
|
31009
31366
|
startTime: eventTime(event.timestamp),
|
|
31010
31367
|
event: { metadata }
|
|
31011
|
-
}
|
|
31368
|
+
};
|
|
31369
|
+
const runSpan = event.runId ? this.runsById.get(event.runId)?.span : void 0;
|
|
31370
|
+
const span = event.operationKind === "prompt" && (!parent || parent === runSpan) ? startFlueRootSpan(args) : startFlueSpan(parent, args);
|
|
31012
31371
|
this.operationsById.set(event.operationId, { metadata, span });
|
|
31013
31372
|
}
|
|
31014
31373
|
handleOperation(event) {
|
|
@@ -31023,6 +31382,11 @@ var FlueObserveBridge = class {
|
|
|
31023
31382
|
...event.isError !== void 0 ? { "flue.is_error": event.isError } : {},
|
|
31024
31383
|
...event.usage ? { "flue.usage": event.usage } : {}
|
|
31025
31384
|
};
|
|
31385
|
+
const input = flueOperationInput(event);
|
|
31386
|
+
if (!state.loggedInput && input !== void 0) {
|
|
31387
|
+
safeLog3(state.span, { input });
|
|
31388
|
+
state.loggedInput = true;
|
|
31389
|
+
}
|
|
31026
31390
|
this.finishPendingChildrenForOperation(event, output);
|
|
31027
31391
|
safeLog3(state.span, {
|
|
31028
31392
|
...event.isError ? { error: toLoggedError(event.errorInfo ?? event.error) } : {},
|
|
@@ -31039,6 +31403,8 @@ var FlueObserveBridge = class {
|
|
|
31039
31403
|
return;
|
|
31040
31404
|
}
|
|
31041
31405
|
const input = flueTurnRequestInput(event);
|
|
31406
|
+
const operation = event.operationId ? this.operationsById.get(event.operationId) : void 0;
|
|
31407
|
+
const turnInput = prepareFlueTurnInput(event, input, operation);
|
|
31042
31408
|
const model = flueTurnRequestModel(event);
|
|
31043
31409
|
const provider = flueTurnRequestProvider(event);
|
|
31044
31410
|
const api = flueTurnRequestApi(event);
|
|
@@ -31051,8 +31417,7 @@ var FlueObserveBridge = class {
|
|
|
31051
31417
|
...provider ? { "flue.provider": provider } : {},
|
|
31052
31418
|
...event.purpose ? { "flue.turn_purpose": event.purpose } : {},
|
|
31053
31419
|
...reasoning ? { reasoning } : {},
|
|
31054
|
-
...
|
|
31055
|
-
...input?.tools ? { tools: input.tools } : {}
|
|
31420
|
+
...turnInput.metadata
|
|
31056
31421
|
};
|
|
31057
31422
|
const parent = this.parentSpanForTurn(event);
|
|
31058
31423
|
const span = startFlueSpan(parent, {
|
|
@@ -31060,11 +31425,14 @@ var FlueObserveBridge = class {
|
|
|
31060
31425
|
spanAttributes: { type: "llm" /* LLM */ },
|
|
31061
31426
|
startTime: eventTime(event.timestamp),
|
|
31062
31427
|
event: {
|
|
31063
|
-
input:
|
|
31428
|
+
input: turnInput.messages,
|
|
31064
31429
|
metadata
|
|
31065
31430
|
}
|
|
31066
31431
|
});
|
|
31067
|
-
this.logOperationInput(
|
|
31432
|
+
this.logOperationInput(
|
|
31433
|
+
event.operationId,
|
|
31434
|
+
latestUserMessageInput(input?.messages)
|
|
31435
|
+
);
|
|
31068
31436
|
this.turnsByKey.set(key, { metadata, span });
|
|
31069
31437
|
}
|
|
31070
31438
|
handleTurn(event) {
|
|
@@ -31311,16 +31679,20 @@ var FlueObserveBridge = class {
|
|
|
31311
31679
|
}
|
|
31312
31680
|
startSyntheticOperation(event) {
|
|
31313
31681
|
const metadata = {
|
|
31682
|
+
...event.runId ? this.runsById.get(event.runId)?.metadata : {},
|
|
31314
31683
|
...extractEventMetadata(event),
|
|
31315
31684
|
"flue.operation": event.operationKind,
|
|
31316
31685
|
provider: "flue"
|
|
31317
31686
|
};
|
|
31318
|
-
const
|
|
31687
|
+
const args = {
|
|
31319
31688
|
name: `flue.${event.operationKind}`,
|
|
31320
31689
|
spanAttributes: { type: "task" /* TASK */ },
|
|
31321
31690
|
startTime: eventTime(event.timestamp),
|
|
31322
31691
|
event: { metadata }
|
|
31323
|
-
}
|
|
31692
|
+
};
|
|
31693
|
+
const parent = this.parentSpanForEvent(event);
|
|
31694
|
+
const runSpan = event.runId ? this.runsById.get(event.runId)?.span : void 0;
|
|
31695
|
+
const span = event.operationKind === "prompt" && (!parent || parent === runSpan) ? startFlueRootSpan(args) : startFlueSpan(parent, args);
|
|
31324
31696
|
return { metadata, span };
|
|
31325
31697
|
}
|
|
31326
31698
|
startSyntheticTurn(event) {
|
|
@@ -31499,6 +31871,71 @@ function flueRunInput(event) {
|
|
|
31499
31871
|
function flueTurnRequestInput(event) {
|
|
31500
31872
|
return event.request?.input ?? event.input;
|
|
31501
31873
|
}
|
|
31874
|
+
function prepareFlueTurnInput(event, input, operation) {
|
|
31875
|
+
const messages = input?.messages;
|
|
31876
|
+
const tracksUserTurn = event.purpose === "agent" && operation?.metadata["flue.operation"] === "prompt" && Array.isArray(messages);
|
|
31877
|
+
if (!tracksUserTurn) {
|
|
31878
|
+
return {
|
|
31879
|
+
messages,
|
|
31880
|
+
metadata: {
|
|
31881
|
+
...input?.systemPrompt ? { "flue.system_prompt": input.systemPrompt } : {},
|
|
31882
|
+
...input?.tools ? { tools: input.tools } : {}
|
|
31883
|
+
}
|
|
31884
|
+
};
|
|
31885
|
+
}
|
|
31886
|
+
const previous = operation.turnInputState;
|
|
31887
|
+
const previousMessageCount = previous?.messageCount ?? 0;
|
|
31888
|
+
const boundaryFingerprint = messages.length > 0 ? fingerprintJsonValue(messages[messages.length - 1]) : void 0;
|
|
31889
|
+
const continuesPreviousInput = previous !== void 0 && messages.length >= previousMessageCount && (previousMessageCount === 0 || previous.boundaryFingerprint !== void 0 && previous.boundaryFingerprint === fingerprintJsonValue(messages[previousMessageCount - 1]));
|
|
31890
|
+
const inputMode = previous === void 0 ? "full" : continuesPreviousInput ? "delta" : "reset";
|
|
31891
|
+
const systemPromptFingerprint = fingerprintJsonValue(input?.systemPrompt);
|
|
31892
|
+
const toolsFingerprint = fingerprintJsonValue(input?.tools);
|
|
31893
|
+
operation.turnInputState = {
|
|
31894
|
+
...boundaryFingerprint !== void 0 ? { boundaryFingerprint } : {},
|
|
31895
|
+
messageCount: messages.length,
|
|
31896
|
+
...systemPromptFingerprint !== void 0 ? { systemPromptFingerprint } : {},
|
|
31897
|
+
...toolsFingerprint !== void 0 ? { toolsFingerprint } : {}
|
|
31898
|
+
};
|
|
31899
|
+
return {
|
|
31900
|
+
messages: continuesPreviousInput ? messages.slice(previousMessageCount) : messages,
|
|
31901
|
+
metadata: {
|
|
31902
|
+
"flue.input_mode": inputMode,
|
|
31903
|
+
...continuesPreviousInput ? { "flue.input_message_offset": previousMessageCount } : {},
|
|
31904
|
+
...input?.systemPrompt && (!continuesPreviousInput || systemPromptFingerprint !== previous?.systemPromptFingerprint) ? { "flue.system_prompt": input.systemPrompt } : {},
|
|
31905
|
+
...input?.tools && (!continuesPreviousInput || toolsFingerprint !== previous?.toolsFingerprint) ? { tools: input.tools } : {}
|
|
31906
|
+
}
|
|
31907
|
+
};
|
|
31908
|
+
}
|
|
31909
|
+
function fingerprintJsonValue(value) {
|
|
31910
|
+
try {
|
|
31911
|
+
const serialized = JSON.stringify(value);
|
|
31912
|
+
if (serialized === void 0) {
|
|
31913
|
+
return void 0;
|
|
31914
|
+
}
|
|
31915
|
+
let hash = 2166136261;
|
|
31916
|
+
for (let i = 0; i < serialized.length; i++) {
|
|
31917
|
+
hash = Math.imul(hash ^ serialized.charCodeAt(i), 16777619);
|
|
31918
|
+
}
|
|
31919
|
+
return `${serialized.length}:${hash >>> 0}`;
|
|
31920
|
+
} catch {
|
|
31921
|
+
return void 0;
|
|
31922
|
+
}
|
|
31923
|
+
}
|
|
31924
|
+
function latestUserMessageInput(messages) {
|
|
31925
|
+
if (!messages) {
|
|
31926
|
+
return void 0;
|
|
31927
|
+
}
|
|
31928
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
31929
|
+
const message = messages[i];
|
|
31930
|
+
if (isObjectLike(message) && Reflect.get(message, "role") === "user") {
|
|
31931
|
+
return [message];
|
|
31932
|
+
}
|
|
31933
|
+
}
|
|
31934
|
+
return void 0;
|
|
31935
|
+
}
|
|
31936
|
+
function flueOperationInput(event) {
|
|
31937
|
+
return typeof event.agentInput?.text === "string" ? [{ content: event.agentInput.text, role: "user" }] : void 0;
|
|
31938
|
+
}
|
|
31502
31939
|
function flueTurnRequestModel(event) {
|
|
31503
31940
|
return event.request?.requestedModel ?? event.request?.model ?? event.model;
|
|
31504
31941
|
}
|
|
@@ -31656,6 +32093,21 @@ function startFlueSpan(parent, args) {
|
|
|
31656
32093
|
withSpanInstrumentationName(args, INSTRUMENTATION_NAMES.FLUE)
|
|
31657
32094
|
);
|
|
31658
32095
|
}
|
|
32096
|
+
function startFlueRootSpan(args) {
|
|
32097
|
+
const state = _internalGetGlobalState();
|
|
32098
|
+
const spanId = state.idGenerator.getSpanId();
|
|
32099
|
+
const rootSpanId = state.idGenerator.shareRootSpanId() ? spanId : state.idGenerator.getTraceId();
|
|
32100
|
+
return withCurrent(
|
|
32101
|
+
NOOP_SPAN,
|
|
32102
|
+
() => startSpan({
|
|
32103
|
+
...withSpanInstrumentationName(args, INSTRUMENTATION_NAMES.FLUE),
|
|
32104
|
+
parentSpanIds: { parentSpanIds: [], rootSpanId },
|
|
32105
|
+
spanId,
|
|
32106
|
+
state
|
|
32107
|
+
}),
|
|
32108
|
+
state
|
|
32109
|
+
);
|
|
32110
|
+
}
|
|
31659
32111
|
function runWithCurrentSpanStore(span, next) {
|
|
31660
32112
|
const state = _internalGetGlobalState();
|
|
31661
32113
|
const contextManager = state?.contextManager;
|
|
@@ -33555,8 +34007,7 @@ function startAgentStream(event, activeChildParents) {
|
|
|
33555
34007
|
...extractAgentMetadata2(agent),
|
|
33556
34008
|
...extractModelMetadata3(model),
|
|
33557
34009
|
"strands.operation": "Agent.stream",
|
|
33558
|
-
provider: extractProvider(model)
|
|
33559
|
-
...event.moduleVersion ? { "strands_agent_sdk.version": event.moduleVersion } : {}
|
|
34010
|
+
provider: extractProvider(model)
|
|
33560
34011
|
};
|
|
33561
34012
|
const parentSpan = agent ? getOnlyChildParent(activeChildParents, agent) : void 0;
|
|
33562
34013
|
const attachmentCache = createStrandsAttachmentCache();
|
|
@@ -33606,8 +34057,7 @@ function startMultiAgentStream(event, operation, activeChildParents) {
|
|
|
33606
34057
|
const metadata = {
|
|
33607
34058
|
"strands.operation": operation,
|
|
33608
34059
|
provider: "strands",
|
|
33609
|
-
...orchestrator?.id ? { "strands.orchestrator.id": orchestrator.id } : {}
|
|
33610
|
-
...event.moduleVersion ? { "strands_agent_sdk.version": event.moduleVersion } : {}
|
|
34060
|
+
...orchestrator?.id ? { "strands.orchestrator.id": orchestrator.id } : {}
|
|
33611
34061
|
};
|
|
33612
34062
|
const parentSpan = orchestrator ? getOnlyChildParent(activeChildParents, orchestrator) : void 0;
|
|
33613
34063
|
const input = processStrandsInputAttachments(event.arguments[0]);
|
|
@@ -34393,6 +34843,320 @@ function logInstrumentationError5(context, error) {
|
|
|
34393
34843
|
debugLogger.debug(`${context}:`, error);
|
|
34394
34844
|
}
|
|
34395
34845
|
|
|
34846
|
+
// src/instrumentation/plugins/voyageai-channels.ts
|
|
34847
|
+
var voyageAIChannels = defineChannels(
|
|
34848
|
+
"voyageai",
|
|
34849
|
+
{
|
|
34850
|
+
embed: channel({
|
|
34851
|
+
channelName: "embed",
|
|
34852
|
+
kind: "async"
|
|
34853
|
+
}),
|
|
34854
|
+
multimodalEmbed: channel({
|
|
34855
|
+
channelName: "multimodalEmbed",
|
|
34856
|
+
kind: "async"
|
|
34857
|
+
}),
|
|
34858
|
+
rerank: channel({
|
|
34859
|
+
channelName: "rerank",
|
|
34860
|
+
kind: "async"
|
|
34861
|
+
}),
|
|
34862
|
+
contextualizedEmbed: channel({
|
|
34863
|
+
channelName: "contextualizedEmbed",
|
|
34864
|
+
kind: "async"
|
|
34865
|
+
})
|
|
34866
|
+
},
|
|
34867
|
+
{ instrumentationName: INSTRUMENTATION_NAMES.VOYAGEAI }
|
|
34868
|
+
);
|
|
34869
|
+
|
|
34870
|
+
// src/instrumentation/plugins/voyageai-plugin.ts
|
|
34871
|
+
var RERANK_METADATA_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
34872
|
+
"model",
|
|
34873
|
+
"returnDocuments",
|
|
34874
|
+
"topK",
|
|
34875
|
+
"truncation"
|
|
34876
|
+
]);
|
|
34877
|
+
var VoyageAIPlugin = class extends BasePlugin {
|
|
34878
|
+
onEnable() {
|
|
34879
|
+
this.unsubscribers.push(
|
|
34880
|
+
interceptVoyageAICall(
|
|
34881
|
+
voyageAIChannels.embed,
|
|
34882
|
+
"voyageai.embed",
|
|
34883
|
+
extractTextEmbeddingInput,
|
|
34884
|
+
summarizeEmbeddingOutput,
|
|
34885
|
+
extractEmbeddingUsageMetrics
|
|
34886
|
+
),
|
|
34887
|
+
interceptVoyageAICall(
|
|
34888
|
+
voyageAIChannels.multimodalEmbed,
|
|
34889
|
+
"voyageai.multimodalEmbed",
|
|
34890
|
+
extractMultimodalEmbeddingInput,
|
|
34891
|
+
summarizeEmbeddingOutput,
|
|
34892
|
+
extractEmbeddingUsageMetrics
|
|
34893
|
+
),
|
|
34894
|
+
interceptVoyageAICall(
|
|
34895
|
+
voyageAIChannels.rerank,
|
|
34896
|
+
"voyageai.rerank",
|
|
34897
|
+
extractRerankInput,
|
|
34898
|
+
summarizeRerankOutput
|
|
34899
|
+
),
|
|
34900
|
+
interceptVoyageAICall(
|
|
34901
|
+
voyageAIChannels.contextualizedEmbed,
|
|
34902
|
+
"voyageai.contextualizedEmbed",
|
|
34903
|
+
extractContextualizedEmbeddingInput,
|
|
34904
|
+
summarizeContextualizedEmbeddingOutput,
|
|
34905
|
+
extractEmbeddingUsageMetrics
|
|
34906
|
+
)
|
|
34907
|
+
);
|
|
34908
|
+
}
|
|
34909
|
+
onDisable() {
|
|
34910
|
+
this.unsubscribers = unsubscribeAll(this.unsubscribers);
|
|
34911
|
+
}
|
|
34912
|
+
};
|
|
34913
|
+
function interceptVoyageAICall(channel2, name, extractInput2, extractOutput2, extractMetrics2 = extractUsageMetrics3) {
|
|
34914
|
+
return channel2.intercept((target, thisArg, args) => {
|
|
34915
|
+
const invokeTarget = () => Reflect.apply(target, thisArg, args);
|
|
34916
|
+
if (isAutoInstrumentationSuppressed()) {
|
|
34917
|
+
return invokeTarget();
|
|
34918
|
+
}
|
|
34919
|
+
let span;
|
|
34920
|
+
try {
|
|
34921
|
+
const { input, metadata } = extractInput2(args);
|
|
34922
|
+
span = startSpan(
|
|
34923
|
+
withSpanInstrumentationName(
|
|
34924
|
+
{
|
|
34925
|
+
event: { input, metadata },
|
|
34926
|
+
name,
|
|
34927
|
+
spanAttributes: { type: "llm" /* LLM */ }
|
|
34928
|
+
},
|
|
34929
|
+
INSTRUMENTATION_NAMES.VOYAGEAI
|
|
34930
|
+
)
|
|
34931
|
+
);
|
|
34932
|
+
} catch (error) {
|
|
34933
|
+
debugLogger.error(`Error starting span for ${name}:`, error);
|
|
34934
|
+
return invokeTarget();
|
|
34935
|
+
}
|
|
34936
|
+
let result;
|
|
34937
|
+
try {
|
|
34938
|
+
result = withCurrent(
|
|
34939
|
+
span,
|
|
34940
|
+
() => runWithAutoInstrumentationSuppressed(invokeTarget)
|
|
34941
|
+
);
|
|
34942
|
+
} catch (error) {
|
|
34943
|
+
finishVoyageAISpan(span, name, () => span.log({ error }));
|
|
34944
|
+
throw error;
|
|
34945
|
+
}
|
|
34946
|
+
void Promise.resolve(result).then(
|
|
34947
|
+
(value) => finishVoyageAISpan(span, name, () => {
|
|
34948
|
+
const metadata = extractResponseMetadata3(value);
|
|
34949
|
+
span.log({
|
|
34950
|
+
output: extractOutput2(value),
|
|
34951
|
+
...metadata ? { metadata } : {},
|
|
34952
|
+
metrics: extractMetrics2(value)
|
|
34953
|
+
});
|
|
34954
|
+
}),
|
|
34955
|
+
(error) => finishVoyageAISpan(span, name, () => span.log({ error }))
|
|
34956
|
+
);
|
|
34957
|
+
return result;
|
|
34958
|
+
});
|
|
34959
|
+
}
|
|
34960
|
+
function finishVoyageAISpan(span, name, log2) {
|
|
34961
|
+
try {
|
|
34962
|
+
log2();
|
|
34963
|
+
} catch (error) {
|
|
34964
|
+
debugLogger.error(`Error logging span for ${name}:`, error);
|
|
34965
|
+
}
|
|
34966
|
+
try {
|
|
34967
|
+
span.end();
|
|
34968
|
+
} catch (error) {
|
|
34969
|
+
debugLogger.error(`Error ending span for ${name}:`, error);
|
|
34970
|
+
}
|
|
34971
|
+
}
|
|
34972
|
+
function getRequestArg2(args) {
|
|
34973
|
+
if (Array.isArray(args)) {
|
|
34974
|
+
return isObject(args[0]) ? args[0] : void 0;
|
|
34975
|
+
}
|
|
34976
|
+
if (!isObject(args)) {
|
|
34977
|
+
return void 0;
|
|
34978
|
+
}
|
|
34979
|
+
const firstArg = Reflect.get(args, "0");
|
|
34980
|
+
return isObject(firstArg) ? firstArg : void 0;
|
|
34981
|
+
}
|
|
34982
|
+
function pickMetadata(request, allowlist) {
|
|
34983
|
+
const metadata = {};
|
|
34984
|
+
if (request) {
|
|
34985
|
+
for (const key of allowlist) {
|
|
34986
|
+
if (!Object.hasOwn(request, key)) {
|
|
34987
|
+
continue;
|
|
34988
|
+
}
|
|
34989
|
+
const value = request[key];
|
|
34990
|
+
if (value !== void 0) {
|
|
34991
|
+
metadata[key] = value;
|
|
34992
|
+
}
|
|
34993
|
+
}
|
|
34994
|
+
}
|
|
34995
|
+
return {
|
|
34996
|
+
...metadata,
|
|
34997
|
+
provider: "voyage"
|
|
34998
|
+
};
|
|
34999
|
+
}
|
|
35000
|
+
function buildEmbeddingInput(inputs, request) {
|
|
35001
|
+
const outputDimensions = request?.outputDimension;
|
|
35002
|
+
return {
|
|
35003
|
+
inputs,
|
|
35004
|
+
...typeof outputDimensions === "number" && Number.isFinite(outputDimensions) ? { output_dimensions: outputDimensions } : {}
|
|
35005
|
+
};
|
|
35006
|
+
}
|
|
35007
|
+
function embeddingMetadata(request) {
|
|
35008
|
+
return {
|
|
35009
|
+
...typeof request?.model === "string" ? { model: request.model } : {},
|
|
35010
|
+
provider: "voyage"
|
|
35011
|
+
};
|
|
35012
|
+
}
|
|
35013
|
+
function extractTextEmbeddingInput(args) {
|
|
35014
|
+
const request = getRequestArg2(args);
|
|
35015
|
+
const rawInput = request?.input;
|
|
35016
|
+
const values = Array.isArray(rawInput) ? rawInput : [rawInput];
|
|
35017
|
+
return {
|
|
35018
|
+
input: buildEmbeddingInput(
|
|
35019
|
+
values.flatMap(
|
|
35020
|
+
(value) => typeof value === "string" ? [{ content: value }] : []
|
|
35021
|
+
),
|
|
35022
|
+
request
|
|
35023
|
+
),
|
|
35024
|
+
metadata: embeddingMetadata(request)
|
|
35025
|
+
};
|
|
35026
|
+
}
|
|
35027
|
+
function extractContextualizedEmbeddingInput(args) {
|
|
35028
|
+
const request = getRequestArg2(args);
|
|
35029
|
+
const rawInputs = request?.inputs;
|
|
35030
|
+
const values = Array.isArray(rawInputs) ? rawInputs.flatMap((value) => Array.isArray(value) ? value : [value]) : [];
|
|
35031
|
+
return {
|
|
35032
|
+
input: buildEmbeddingInput(
|
|
35033
|
+
values.flatMap(
|
|
35034
|
+
(value) => typeof value === "string" ? [{ content: value }] : []
|
|
35035
|
+
),
|
|
35036
|
+
request
|
|
35037
|
+
),
|
|
35038
|
+
metadata: embeddingMetadata(request)
|
|
35039
|
+
};
|
|
35040
|
+
}
|
|
35041
|
+
function extractMultimodalEmbeddingInput(args) {
|
|
35042
|
+
const request = getRequestArg2(args);
|
|
35043
|
+
const rawInputs = request?.inputs;
|
|
35044
|
+
const inputs = Array.isArray(rawInputs) ? rawInputs.map((rawInput) => {
|
|
35045
|
+
const rawContent = isObject(rawInput) ? rawInput.content : void 0;
|
|
35046
|
+
return {
|
|
35047
|
+
content: Array.isArray(rawContent) ? rawContent.flatMap(normalizeMultimodalContentPart) : []
|
|
35048
|
+
};
|
|
35049
|
+
}) : [];
|
|
35050
|
+
const input = buildEmbeddingInput(inputs, request);
|
|
35051
|
+
const processedInput = processInputAttachments(input);
|
|
35052
|
+
return {
|
|
35053
|
+
input: hasInlineEmbeddingMedia(processedInput) ? input : processedInput,
|
|
35054
|
+
metadata: embeddingMetadata(request)
|
|
35055
|
+
};
|
|
35056
|
+
}
|
|
35057
|
+
function normalizeMultimodalContentPart(part) {
|
|
35058
|
+
if (!isObject(part) || typeof part.type !== "string") {
|
|
35059
|
+
return [];
|
|
35060
|
+
}
|
|
35061
|
+
if (part.type === "text") {
|
|
35062
|
+
return typeof part.text === "string" ? [{ type: "text", text: part.text }] : [];
|
|
35063
|
+
}
|
|
35064
|
+
const camelCaseField = {
|
|
35065
|
+
image_base64: "imageBase64",
|
|
35066
|
+
image_url: "imageUrl",
|
|
35067
|
+
video_base64: "videoBase64",
|
|
35068
|
+
video_url: "videoUrl"
|
|
35069
|
+
};
|
|
35070
|
+
const field = camelCaseField[part.type];
|
|
35071
|
+
if (!field) {
|
|
35072
|
+
return [];
|
|
35073
|
+
}
|
|
35074
|
+
const data = typeof part[field] === "string" ? part[field] : part[part.type];
|
|
35075
|
+
if (typeof data !== "string") {
|
|
35076
|
+
return [];
|
|
35077
|
+
}
|
|
35078
|
+
return part.type.startsWith("image_") ? [{ type: "image_url", image_url: { url: data } }] : [{ type: "file", file: { file_data: data } }];
|
|
35079
|
+
}
|
|
35080
|
+
function hasInlineEmbeddingMedia(input) {
|
|
35081
|
+
return input.inputs.some(
|
|
35082
|
+
({ content }) => Array.isArray(content) ? content.some((part) => {
|
|
35083
|
+
const value = part.type === "image_url" ? part.image_url.url : part.type === "file" ? part.file.file_data : void 0;
|
|
35084
|
+
return typeof value === "string" && value.startsWith("data:");
|
|
35085
|
+
}) : false
|
|
35086
|
+
);
|
|
35087
|
+
}
|
|
35088
|
+
function extractRerankInput(args) {
|
|
35089
|
+
const request = getRequestArg2(args);
|
|
35090
|
+
const documents = request?.documents;
|
|
35091
|
+
return {
|
|
35092
|
+
input: {
|
|
35093
|
+
documents,
|
|
35094
|
+
query: request?.query
|
|
35095
|
+
},
|
|
35096
|
+
metadata: {
|
|
35097
|
+
...pickMetadata(request, RERANK_METADATA_ALLOWLIST),
|
|
35098
|
+
...Array.isArray(documents) ? { document_count: documents.length } : {}
|
|
35099
|
+
}
|
|
35100
|
+
};
|
|
35101
|
+
}
|
|
35102
|
+
function extractResponseMetadata3(result) {
|
|
35103
|
+
if (!isObject(result)) {
|
|
35104
|
+
return void 0;
|
|
35105
|
+
}
|
|
35106
|
+
const rawResponse = isObject(result.rawResponse) ? result.rawResponse : void 0;
|
|
35107
|
+
const model = typeof result.model === "string" ? result.model : typeof rawResponse?.model === "string" ? rawResponse.model : void 0;
|
|
35108
|
+
return model ? { model } : void 0;
|
|
35109
|
+
}
|
|
35110
|
+
function summarizeEmbeddingOutput(result) {
|
|
35111
|
+
return {
|
|
35112
|
+
count: isObject(result) && Array.isArray(result.data) ? result.data.length : 0
|
|
35113
|
+
};
|
|
35114
|
+
}
|
|
35115
|
+
function summarizeRerankOutput(result) {
|
|
35116
|
+
if (!isObject(result) || !Array.isArray(result.data)) {
|
|
35117
|
+
return void 0;
|
|
35118
|
+
}
|
|
35119
|
+
return result.data.slice(0, 100).map((item) => ({
|
|
35120
|
+
index: isObject(item) ? item.index : void 0,
|
|
35121
|
+
relevance_score: isObject(item) ? (typeof item.relevanceScore === "number" ? item.relevanceScore : item.relevance_score) ?? null : null
|
|
35122
|
+
}));
|
|
35123
|
+
}
|
|
35124
|
+
function summarizeContextualizedEmbeddingOutput(result) {
|
|
35125
|
+
if (!isObject(result)) {
|
|
35126
|
+
return { count: 0 };
|
|
35127
|
+
}
|
|
35128
|
+
if (Array.isArray(result.results)) {
|
|
35129
|
+
return {
|
|
35130
|
+
count: result.results.reduce(
|
|
35131
|
+
(count, item) => count + (isObject(item) && Array.isArray(item.embeddings) ? item.embeddings.length : 0),
|
|
35132
|
+
0
|
|
35133
|
+
)
|
|
35134
|
+
};
|
|
35135
|
+
}
|
|
35136
|
+
if (!Array.isArray(result.data)) {
|
|
35137
|
+
return { count: 0 };
|
|
35138
|
+
}
|
|
35139
|
+
return {
|
|
35140
|
+
count: result.data.reduce(
|
|
35141
|
+
(count, item) => count + (isObject(item) && Array.isArray(item.data) ? item.data.length : 0),
|
|
35142
|
+
0
|
|
35143
|
+
)
|
|
35144
|
+
};
|
|
35145
|
+
}
|
|
35146
|
+
function extractEmbeddingUsageMetrics(result) {
|
|
35147
|
+
const metrics = extractUsageMetrics3(result);
|
|
35148
|
+
return typeof metrics.tokens === "number" ? { prompt_tokens: metrics.tokens, tokens: metrics.tokens } : {};
|
|
35149
|
+
}
|
|
35150
|
+
function extractUsageMetrics3(result) {
|
|
35151
|
+
if (!isObject(result)) {
|
|
35152
|
+
return {};
|
|
35153
|
+
}
|
|
35154
|
+
const rawResponse = isObject(result.rawResponse) ? result.rawResponse : void 0;
|
|
35155
|
+
const usage = isObject(result.usage) ? result.usage : isObject(rawResponse?.usage) ? rawResponse.usage : void 0;
|
|
35156
|
+
const tokens = typeof result.totalTokens === "number" ? result.totalTokens : usage?.totalTokens ?? usage?.total_tokens;
|
|
35157
|
+
return typeof tokens === "number" && Number.isFinite(tokens) && tokens >= 0 ? { tokens } : {};
|
|
35158
|
+
}
|
|
35159
|
+
|
|
34396
35160
|
// src/instrumentation/plugins/cloudflare-ai-chat-channels.ts
|
|
34397
35161
|
var cloudflareAIChatChannels = defineChannels(
|
|
34398
35162
|
"@cloudflare/ai-chat",
|
|
@@ -34948,6 +35712,7 @@ var BraintrustPlugin = class extends BasePlugin {
|
|
|
34948
35712
|
langSmithPlugin = null;
|
|
34949
35713
|
piCodingAgentPlugin = null;
|
|
34950
35714
|
strandsAgentSDKPlugin = null;
|
|
35715
|
+
voyageAIPlugin = null;
|
|
34951
35716
|
cloudflareAIChatPlugin = null;
|
|
34952
35717
|
cloudflareAgentsPlugin = null;
|
|
34953
35718
|
constructor(config = {}) {
|
|
@@ -35022,6 +35787,10 @@ var BraintrustPlugin = class extends BasePlugin {
|
|
|
35022
35787
|
this.coherePlugin = new CoherePlugin();
|
|
35023
35788
|
this.coherePlugin.enable();
|
|
35024
35789
|
}
|
|
35790
|
+
if (integrations.voyageai !== false) {
|
|
35791
|
+
this.voyageAIPlugin = new VoyageAIPlugin();
|
|
35792
|
+
this.voyageAIPlugin.enable();
|
|
35793
|
+
}
|
|
35025
35794
|
if (integrations.groq !== false) {
|
|
35026
35795
|
this.groqPlugin = new GroqPlugin();
|
|
35027
35796
|
this.groqPlugin.enable();
|
|
@@ -35138,6 +35907,10 @@ var BraintrustPlugin = class extends BasePlugin {
|
|
|
35138
35907
|
this.coherePlugin.disable();
|
|
35139
35908
|
this.coherePlugin = null;
|
|
35140
35909
|
}
|
|
35910
|
+
if (this.voyageAIPlugin) {
|
|
35911
|
+
this.voyageAIPlugin.disable();
|
|
35912
|
+
this.voyageAIPlugin = null;
|
|
35913
|
+
}
|
|
35141
35914
|
if (this.groqPlugin) {
|
|
35142
35915
|
this.groqPlugin.disable();
|
|
35143
35916
|
this.groqPlugin = null;
|
|
@@ -35257,7 +36030,10 @@ var envIntegrationAliases = {
|
|
|
35257
36030
|
"langchain-js": "langchain",
|
|
35258
36031
|
"@langchain": "langchain",
|
|
35259
36032
|
langgraph: "langgraph",
|
|
35260
|
-
langsmith: "langsmith"
|
|
36033
|
+
langsmith: "langsmith",
|
|
36034
|
+
voyage: "voyageai",
|
|
36035
|
+
"voyage-ai": "voyageai",
|
|
36036
|
+
voyageai: "voyageai"
|
|
35261
36037
|
};
|
|
35262
36038
|
function getDefaultInstrumentationIntegrations() {
|
|
35263
36039
|
return {
|
|
@@ -35292,6 +36068,7 @@ function getDefaultInstrumentationIntegrations() {
|
|
|
35292
36068
|
langchain: true,
|
|
35293
36069
|
langgraph: true,
|
|
35294
36070
|
langsmith: true,
|
|
36071
|
+
voyageai: true,
|
|
35295
36072
|
piCodingAgent: true,
|
|
35296
36073
|
strandsAgentSDK: true,
|
|
35297
36074
|
cloudflareAgents: true
|
|
@@ -35712,9 +36489,9 @@ function configureNode() {
|
|
|
35712
36489
|
return value;
|
|
35713
36490
|
}
|
|
35714
36491
|
const envPaths = [];
|
|
35715
|
-
for (let dir2 = process.cwd(), depth = 0; depth <= BRAINTRUST_ENV_SEARCH_PARENT_LIMIT; dir2 =
|
|
35716
|
-
envPaths.push(
|
|
35717
|
-
if (
|
|
36492
|
+
for (let dir2 = process.cwd(), depth = 0; depth <= BRAINTRUST_ENV_SEARCH_PARENT_LIMIT; dir2 = path2.dirname(dir2), depth++) {
|
|
36493
|
+
envPaths.push(path2.join(dir2, ".env.braintrust"));
|
|
36494
|
+
if (path2.dirname(dir2) === dir2) {
|
|
35718
36495
|
break;
|
|
35719
36496
|
}
|
|
35720
36497
|
}
|
|
@@ -35756,10 +36533,10 @@ function configureNode() {
|
|
|
35756
36533
|
isomorph_default.processOn = (event, handler) => {
|
|
35757
36534
|
process.on(event, handler);
|
|
35758
36535
|
};
|
|
35759
|
-
isomorph_default.basename =
|
|
36536
|
+
isomorph_default.basename = path2.basename;
|
|
35760
36537
|
isomorph_default.writeln = (text) => process.stdout.write(text + "\n");
|
|
35761
|
-
isomorph_default.pathJoin =
|
|
35762
|
-
isomorph_default.pathDirname =
|
|
36538
|
+
isomorph_default.pathJoin = path2.join;
|
|
36539
|
+
isomorph_default.pathDirname = path2.dirname;
|
|
35763
36540
|
isomorph_default.mkdir = fs.mkdir;
|
|
35764
36541
|
isomorph_default.writeFile = fs.writeFile;
|
|
35765
36542
|
isomorph_default.readFile = fs.readFile;
|
|
@@ -35794,8 +36571,8 @@ function configureNode() {
|
|
|
35794
36571
|
registry.enable();
|
|
35795
36572
|
}
|
|
35796
36573
|
function getNearestBraintrustEnvValue(name) {
|
|
35797
|
-
for (let dir2 = process.cwd(), depth = 0; depth <= BRAINTRUST_ENV_SEARCH_PARENT_LIMIT; dir2 =
|
|
35798
|
-
const envPath =
|
|
36574
|
+
for (let dir2 = process.cwd(), depth = 0; depth <= BRAINTRUST_ENV_SEARCH_PARENT_LIMIT; dir2 = path2.dirname(dir2), depth++) {
|
|
36575
|
+
const envPath = path2.join(dir2, ".env.braintrust");
|
|
35799
36576
|
try {
|
|
35800
36577
|
const parsed = dotenv.parse(fsSync.readFileSync(envPath, "utf8"));
|
|
35801
36578
|
const value = parsed[name];
|
|
@@ -35805,7 +36582,7 @@ function getNearestBraintrustEnvValue(name) {
|
|
|
35805
36582
|
return void 0;
|
|
35806
36583
|
}
|
|
35807
36584
|
}
|
|
35808
|
-
if (
|
|
36585
|
+
if (path2.dirname(dir2) === dir2) {
|
|
35809
36586
|
break;
|
|
35810
36587
|
}
|
|
35811
36588
|
}
|
|
@@ -37371,8 +38148,8 @@ function validateParametersWithJsonSchema(parameters, schema) {
|
|
|
37371
38148
|
const validate = ajv.compile(schema);
|
|
37372
38149
|
if (!validate(parameters)) {
|
|
37373
38150
|
const errorMessages = validate.errors?.map((err) => {
|
|
37374
|
-
const
|
|
37375
|
-
return `${
|
|
38151
|
+
const path3 = err.instancePath || "root";
|
|
38152
|
+
return `${path3}: ${err.message}`;
|
|
37376
38153
|
}).join(", ");
|
|
37377
38154
|
throw Error(`Invalid parameters: ${errorMessages}`);
|
|
37378
38155
|
}
|
|
@@ -37401,6 +38178,9 @@ function rehydrateRemoteParameters(parameters, schema) {
|
|
|
37401
38178
|
}
|
|
37402
38179
|
|
|
37403
38180
|
// src/framework.ts
|
|
38181
|
+
function BaseExperiment(options = {}) {
|
|
38182
|
+
return { _type: "BaseExperiment", ...options };
|
|
38183
|
+
}
|
|
37404
38184
|
var EvalResultWithSummary = class {
|
|
37405
38185
|
constructor(summary, results) {
|
|
37406
38186
|
this.summary = summary;
|
|
@@ -37461,6 +38241,26 @@ async function getExperimentParametersRef(parameters) {
|
|
|
37461
38241
|
version: resolvedParameters.version
|
|
37462
38242
|
};
|
|
37463
38243
|
}
|
|
38244
|
+
async function _internalInitEvaluatorExperiment(projectName, evaluator, data, options = {}) {
|
|
38245
|
+
if (options.disabled) return null;
|
|
38246
|
+
const { baseExperiment } = callEvaluatorData(data);
|
|
38247
|
+
const parameters = await getExperimentParametersRef(evaluator.parameters);
|
|
38248
|
+
return initExperiment(evaluator.state, {
|
|
38249
|
+
...evaluator.projectId ? { projectId: evaluator.projectId } : { project: projectName },
|
|
38250
|
+
experiment: options.experimentName ?? evaluator.experimentName,
|
|
38251
|
+
description: evaluator.description,
|
|
38252
|
+
metadata: evaluator.metadata,
|
|
38253
|
+
tags: evaluator.tags,
|
|
38254
|
+
isPublic: evaluator.isPublic,
|
|
38255
|
+
update: options.update ?? evaluator.update,
|
|
38256
|
+
baseExperiment: evaluator.baseExperimentName ?? baseExperiment,
|
|
38257
|
+
baseExperimentId: evaluator.baseExperimentId,
|
|
38258
|
+
gitMetadataSettings: evaluator.gitMetadataSettings,
|
|
38259
|
+
repoInfo: evaluator.repoInfo,
|
|
38260
|
+
dataset: Dataset2.isDataset(data) ? data : void 0,
|
|
38261
|
+
parameters
|
|
38262
|
+
});
|
|
38263
|
+
}
|
|
37464
38264
|
function callEvaluatorData(data) {
|
|
37465
38265
|
const dataResult = typeof data === "function" ? data() : data;
|
|
37466
38266
|
let baseExperiment = void 0;
|
|
@@ -37478,6 +38278,48 @@ function isAsyncIterable7(value) {
|
|
|
37478
38278
|
function isIterable(value) {
|
|
37479
38279
|
return typeof value === "object" && value !== null && Symbol.iterator in value && typeof value[Symbol.iterator] === "function";
|
|
37480
38280
|
}
|
|
38281
|
+
async function _internalResolveEvaluatorData(evaluator, experiment) {
|
|
38282
|
+
if (typeof evaluator.data === "string") {
|
|
38283
|
+
throw new Error("Unimplemented: string data paths");
|
|
38284
|
+
}
|
|
38285
|
+
let dataResult = typeof evaluator.data === "function" ? evaluator.data() : evaluator.data;
|
|
38286
|
+
if ("_type" in dataResult) {
|
|
38287
|
+
if (dataResult._type !== "BaseExperiment") {
|
|
38288
|
+
throw new Error("Invalid _type");
|
|
38289
|
+
}
|
|
38290
|
+
if (!experiment) {
|
|
38291
|
+
throw new Error(
|
|
38292
|
+
"Cannot use BaseExperiment() without connecting to Braintrust (you most likely set --no-send-logs)"
|
|
38293
|
+
);
|
|
38294
|
+
}
|
|
38295
|
+
let name = dataResult.name;
|
|
38296
|
+
if (isEmpty2(name)) {
|
|
38297
|
+
const baseExperiment = await experiment.fetchBaseExperiment();
|
|
38298
|
+
if (!baseExperiment) {
|
|
38299
|
+
throw new Error("BaseExperiment() failed to fetch base experiment");
|
|
38300
|
+
}
|
|
38301
|
+
name = baseExperiment.name;
|
|
38302
|
+
}
|
|
38303
|
+
dataResult = initExperiment(evaluator.state, {
|
|
38304
|
+
...evaluator.projectId ? { projectId: evaluator.projectId } : { project: evaluator.projectName },
|
|
38305
|
+
experiment: name,
|
|
38306
|
+
open: true
|
|
38307
|
+
}).asDataset();
|
|
38308
|
+
}
|
|
38309
|
+
const resolvedDataResult = dataResult instanceof Promise ? await dataResult : dataResult;
|
|
38310
|
+
if (isAsyncIterable7(resolvedDataResult)) {
|
|
38311
|
+
return resolvedDataResult;
|
|
38312
|
+
}
|
|
38313
|
+
if (Array.isArray(resolvedDataResult) || isIterable(resolvedDataResult)) {
|
|
38314
|
+
const iterable = resolvedDataResult;
|
|
38315
|
+
return (async function* () {
|
|
38316
|
+
for (const datum of iterable) yield datum;
|
|
38317
|
+
})();
|
|
38318
|
+
}
|
|
38319
|
+
throw new Error(
|
|
38320
|
+
"Evaluator data must be an array, iterable, or async iterable"
|
|
38321
|
+
);
|
|
38322
|
+
}
|
|
37481
38323
|
globalThis._evals = {
|
|
37482
38324
|
functions: [],
|
|
37483
38325
|
prompts: [],
|
|
@@ -37524,25 +38366,13 @@ async function Eval(name, evaluator, reporterOrOpts) {
|
|
|
37524
38366
|
}
|
|
37525
38367
|
const resolvedReporter = options.reporter || defaultReporter;
|
|
37526
38368
|
try {
|
|
37527
|
-
const { data
|
|
37528
|
-
|
|
38369
|
+
const { data } = callEvaluatorData(evaluator.data);
|
|
38370
|
+
const experiment = await _internalInitEvaluatorExperiment(
|
|
38371
|
+
name,
|
|
38372
|
+
evaluator,
|
|
38373
|
+
data,
|
|
38374
|
+
{ disabled: Boolean(options.parent || options.noSendLogs) }
|
|
37529
38375
|
);
|
|
37530
|
-
const parameters = await getExperimentParametersRef(evaluator.parameters);
|
|
37531
|
-
const experiment = options.parent || options.noSendLogs ? null : initExperiment(evaluator.state, {
|
|
37532
|
-
...evaluator.projectId ? { projectId: evaluator.projectId } : { project: name },
|
|
37533
|
-
experiment: evaluator.experimentName,
|
|
37534
|
-
description: evaluator.description,
|
|
37535
|
-
metadata: evaluator.metadata,
|
|
37536
|
-
tags: evaluator.tags,
|
|
37537
|
-
isPublic: evaluator.isPublic,
|
|
37538
|
-
update: evaluator.update,
|
|
37539
|
-
baseExperiment: evaluator.baseExperimentName ?? defaultBaseExperiment,
|
|
37540
|
-
baseExperimentId: evaluator.baseExperimentId,
|
|
37541
|
-
gitMetadataSettings: evaluator.gitMetadataSettings,
|
|
37542
|
-
repoInfo: evaluator.repoInfo,
|
|
37543
|
-
dataset: Dataset2.isDataset(data) ? data : void 0,
|
|
37544
|
-
parameters
|
|
37545
|
-
});
|
|
37546
38376
|
if (experiment && typeof process !== "undefined" && globalThis.BRAINTRUST_CONTEXT_MANAGER !== void 0) {
|
|
37547
38377
|
await experiment._waitForId();
|
|
37548
38378
|
}
|
|
@@ -37611,8 +38441,8 @@ function serializeJSONWithPlainString(v) {
|
|
|
37611
38441
|
}
|
|
37612
38442
|
}
|
|
37613
38443
|
function evaluateFilter(object, filter2) {
|
|
37614
|
-
const { path:
|
|
37615
|
-
const key =
|
|
38444
|
+
const { path: path3, pattern } = filter2;
|
|
38445
|
+
const key = path3.reduce(
|
|
37616
38446
|
(acc, p) => typeof acc === "object" && acc !== null ? (
|
|
37617
38447
|
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
37618
38448
|
acc[p]
|
|
@@ -37630,19 +38460,74 @@ function scorerName(scorer, scorer_idx) {
|
|
|
37630
38460
|
function classifierName(classifier, classifier_idx) {
|
|
37631
38461
|
return classifier.name || `classifier_${classifier_idx}`;
|
|
37632
38462
|
}
|
|
38463
|
+
async function _internalRunEvaluatorTask(task, datum, trialIndex, parameters, span, reportProgress = () => void 0) {
|
|
38464
|
+
const metadata = {
|
|
38465
|
+
..."metadata" in datum ? datum.metadata : {}
|
|
38466
|
+
};
|
|
38467
|
+
const hooks = {
|
|
38468
|
+
meta(value) {
|
|
38469
|
+
Object.assign(metadata, value);
|
|
38470
|
+
},
|
|
38471
|
+
metadata,
|
|
38472
|
+
expected: "expected" in datum ? datum.expected : void 0,
|
|
38473
|
+
span,
|
|
38474
|
+
parameters,
|
|
38475
|
+
reportProgress,
|
|
38476
|
+
trialIndex,
|
|
38477
|
+
tags: [...datum.tags ?? []]
|
|
38478
|
+
};
|
|
38479
|
+
const output = await task(datum.input, hooks);
|
|
38480
|
+
span.log({ output });
|
|
38481
|
+
return {
|
|
38482
|
+
output,
|
|
38483
|
+
metadata: hooks.metadata,
|
|
38484
|
+
tags: hooks.tags ?? []
|
|
38485
|
+
};
|
|
38486
|
+
}
|
|
37633
38487
|
function buildSpanMetadata(results) {
|
|
37634
|
-
return results.length === 1 ? results[0].metadata :
|
|
37635
|
-
(
|
|
37636
|
-
{}
|
|
38488
|
+
return results.length === 1 ? results[0].metadata : Object.fromEntries(
|
|
38489
|
+
results.map((result) => [result.name, result.metadata])
|
|
37637
38490
|
);
|
|
37638
38491
|
}
|
|
37639
38492
|
function buildSpanScores(results) {
|
|
37640
|
-
const scoresRecord =
|
|
37641
|
-
(
|
|
37642
|
-
{}
|
|
38493
|
+
const scoresRecord = Object.fromEntries(
|
|
38494
|
+
results.map((result) => [result.name, result.score])
|
|
37643
38495
|
);
|
|
37644
38496
|
return { resultMetadata: buildSpanMetadata(results), scoresRecord };
|
|
37645
38497
|
}
|
|
38498
|
+
function _internalPrepareEvaluatorScore(scoreValue, name) {
|
|
38499
|
+
if (scoreValue === null) return { results: null };
|
|
38500
|
+
if (Array.isArray(scoreValue)) {
|
|
38501
|
+
for (const score of scoreValue) {
|
|
38502
|
+
if (!(typeof score === "object" && !isEmpty2(score))) {
|
|
38503
|
+
throw new Error(
|
|
38504
|
+
`When returning an array of scores, each score must be a non-empty object. Got: ${JSON.stringify(score)}`
|
|
38505
|
+
);
|
|
38506
|
+
}
|
|
38507
|
+
}
|
|
38508
|
+
}
|
|
38509
|
+
let results;
|
|
38510
|
+
if (Array.isArray(scoreValue)) {
|
|
38511
|
+
results = scoreValue;
|
|
38512
|
+
} else if (typeof scoreValue === "object" && !isEmpty2(scoreValue)) {
|
|
38513
|
+
results = [scoreValue];
|
|
38514
|
+
} else {
|
|
38515
|
+
results = [{ name, score: scoreValue }];
|
|
38516
|
+
}
|
|
38517
|
+
const { resultMetadata, scoresRecord } = buildSpanScores(results);
|
|
38518
|
+
const fields = (score) => {
|
|
38519
|
+
const { metadata: _metadata, name: _name, ...rest } = score;
|
|
38520
|
+
return rest;
|
|
38521
|
+
};
|
|
38522
|
+
return {
|
|
38523
|
+
results,
|
|
38524
|
+
output: results.length === 1 ? fields(results[0]) : Object.fromEntries(
|
|
38525
|
+
results.map((score) => [score.name ?? name, fields(score)])
|
|
38526
|
+
),
|
|
38527
|
+
metadata: resultMetadata,
|
|
38528
|
+
scores: scoresRecord
|
|
38529
|
+
};
|
|
38530
|
+
}
|
|
37646
38531
|
async function runInScorerSpan(rootSpan, spanName, spanType, propagatedEvent, eventInput, fn) {
|
|
37647
38532
|
try {
|
|
37648
38533
|
const value = await rootSpan.traced(fn, {
|
|
@@ -37687,6 +38572,27 @@ function toClassificationItem(c) {
|
|
|
37687
38572
|
...c.metadata !== void 0 ? { metadata: c.metadata } : {}
|
|
37688
38573
|
};
|
|
37689
38574
|
}
|
|
38575
|
+
function _internalPrepareEvaluatorClassification(value, name) {
|
|
38576
|
+
if (value === null) return { results: null };
|
|
38577
|
+
const results = (Array.isArray(value) ? value : [value]).map(
|
|
38578
|
+
(result) => validateClassificationResult(result, name)
|
|
38579
|
+
);
|
|
38580
|
+
const classifications = /* @__PURE__ */ Object.create(null);
|
|
38581
|
+
for (const result of results) {
|
|
38582
|
+
(classifications[result.name] ??= []).push(toClassificationItem(result));
|
|
38583
|
+
}
|
|
38584
|
+
return {
|
|
38585
|
+
results,
|
|
38586
|
+
output: results.length === 1 ? toClassificationItem(results[0]) : Object.fromEntries(
|
|
38587
|
+
results.map((result) => [
|
|
38588
|
+
result.name,
|
|
38589
|
+
toClassificationItem(result)
|
|
38590
|
+
])
|
|
38591
|
+
),
|
|
38592
|
+
metadata: buildSpanMetadata(results),
|
|
38593
|
+
classifications
|
|
38594
|
+
};
|
|
38595
|
+
}
|
|
37690
38596
|
function logScoringFailures(kind, failures, metadata, rootSpan, state) {
|
|
37691
38597
|
if (!failures.length) return [];
|
|
37692
38598
|
const errorMap = Object.fromEntries(
|
|
@@ -37725,54 +38631,14 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
37725
38631
|
(evaluator.state ?? _internalGetGlobalState())?.spanCache?.start();
|
|
37726
38632
|
}
|
|
37727
38633
|
try {
|
|
37728
|
-
if (typeof evaluator.data === "string") {
|
|
37729
|
-
throw new Error("Unimplemented: string data paths");
|
|
37730
|
-
}
|
|
37731
|
-
let dataResult = typeof evaluator.data === "function" ? evaluator.data() : evaluator.data;
|
|
37732
38634
|
parameters = await validateParameters(
|
|
37733
38635
|
parameters ?? {},
|
|
37734
38636
|
evaluator.parameters
|
|
37735
38637
|
);
|
|
37736
|
-
|
|
37737
|
-
|
|
37738
|
-
|
|
37739
|
-
|
|
37740
|
-
if (!experiment) {
|
|
37741
|
-
throw new Error(
|
|
37742
|
-
"Cannot use BaseExperiment() without connecting to Braintrust (you most likely set --no-send-logs)"
|
|
37743
|
-
);
|
|
37744
|
-
}
|
|
37745
|
-
let name = dataResult.name;
|
|
37746
|
-
if (isEmpty2(name)) {
|
|
37747
|
-
const baseExperiment = await experiment.fetchBaseExperiment();
|
|
37748
|
-
if (!baseExperiment) {
|
|
37749
|
-
throw new Error("BaseExperiment() failed to fetch base experiment");
|
|
37750
|
-
}
|
|
37751
|
-
name = baseExperiment.name;
|
|
37752
|
-
}
|
|
37753
|
-
dataResult = initExperiment(evaluator.state, {
|
|
37754
|
-
...evaluator.projectId ? { projectId: evaluator.projectId } : { project: evaluator.projectName },
|
|
37755
|
-
experiment: name,
|
|
37756
|
-
open: true
|
|
37757
|
-
}).asDataset();
|
|
37758
|
-
}
|
|
37759
|
-
const resolvedDataResult = dataResult instanceof Promise ? await dataResult : dataResult;
|
|
37760
|
-
const dataIterable = (() => {
|
|
37761
|
-
if (isAsyncIterable7(resolvedDataResult)) {
|
|
37762
|
-
return resolvedDataResult;
|
|
37763
|
-
}
|
|
37764
|
-
if (Array.isArray(resolvedDataResult) || isIterable(resolvedDataResult)) {
|
|
37765
|
-
const iterable = resolvedDataResult;
|
|
37766
|
-
return (async function* () {
|
|
37767
|
-
for (const datum of iterable) {
|
|
37768
|
-
yield datum;
|
|
37769
|
-
}
|
|
37770
|
-
})();
|
|
37771
|
-
}
|
|
37772
|
-
throw new Error(
|
|
37773
|
-
"Evaluator data must be an array, iterable, or async iterable"
|
|
37774
|
-
);
|
|
37775
|
-
})();
|
|
38638
|
+
const dataIterable = await _internalResolveEvaluatorData(
|
|
38639
|
+
evaluator,
|
|
38640
|
+
experiment
|
|
38641
|
+
);
|
|
37776
38642
|
progressReporter.start(evaluator.evalName, 0);
|
|
37777
38643
|
const experimentIdPromise = experiment ? (async () => {
|
|
37778
38644
|
try {
|
|
@@ -37841,57 +38707,45 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
37841
38707
|
ensureSpansFlushed,
|
|
37842
38708
|
state
|
|
37843
38709
|
}) : void 0;
|
|
37844
|
-
let metadata = {
|
|
37845
|
-
..."metadata" in datum ? datum.metadata : {}
|
|
37846
|
-
};
|
|
38710
|
+
let metadata = {};
|
|
37847
38711
|
const expected = "expected" in datum ? datum.expected : void 0;
|
|
37848
38712
|
let output = void 0;
|
|
37849
38713
|
let error = void 0;
|
|
37850
|
-
let tags = [
|
|
37851
|
-
const scores =
|
|
37852
|
-
const classifications =
|
|
38714
|
+
let tags = [];
|
|
38715
|
+
const scores = /* @__PURE__ */ Object.create(null);
|
|
38716
|
+
const classifications = /* @__PURE__ */ Object.create(null);
|
|
37853
38717
|
const scorerNames = (evaluator.scores ?? []).map(scorerName);
|
|
37854
38718
|
const classifierNames = (evaluator.classifiers ?? []).map(
|
|
37855
38719
|
classifierName
|
|
37856
38720
|
);
|
|
37857
38721
|
let unhandledScores = scorerNames;
|
|
37858
38722
|
try {
|
|
37859
|
-
const
|
|
37860
|
-
|
|
37861
|
-
|
|
37862
|
-
|
|
37863
|
-
|
|
37864
|
-
|
|
37865
|
-
|
|
37866
|
-
|
|
37867
|
-
|
|
37868
|
-
|
|
37869
|
-
|
|
37870
|
-
|
|
37871
|
-
|
|
37872
|
-
|
|
37873
|
-
|
|
37874
|
-
object_type: "task"
|
|
37875
|
-
});
|
|
37876
|
-
},
|
|
37877
|
-
trialIndex,
|
|
37878
|
-
tags
|
|
37879
|
-
};
|
|
37880
|
-
const outputResult = evaluator.task(datum.input, hooksForTask);
|
|
37881
|
-
if (outputResult instanceof Promise) {
|
|
37882
|
-
output = await outputResult;
|
|
37883
|
-
} else {
|
|
37884
|
-
output = outputResult;
|
|
38723
|
+
const taskResult = await rootSpan.traced(
|
|
38724
|
+
(span) => _internalRunEvaluatorTask(
|
|
38725
|
+
evaluator.task,
|
|
38726
|
+
datum,
|
|
38727
|
+
trialIndex,
|
|
38728
|
+
parameters ?? {},
|
|
38729
|
+
span,
|
|
38730
|
+
(event) => {
|
|
38731
|
+
stream?.({
|
|
38732
|
+
...event,
|
|
38733
|
+
id: rootSpan.id,
|
|
38734
|
+
origin: baseEvent.event?.origin,
|
|
38735
|
+
name: evaluator.evalName,
|
|
38736
|
+
object_type: "task"
|
|
38737
|
+
});
|
|
37885
38738
|
}
|
|
37886
|
-
|
|
37887
|
-
span.log({ output });
|
|
37888
|
-
},
|
|
38739
|
+
),
|
|
37889
38740
|
{
|
|
37890
38741
|
name: "task",
|
|
37891
38742
|
spanAttributes: { type: "task" /* TASK */ },
|
|
37892
38743
|
event: { input: datum.input }
|
|
37893
38744
|
}
|
|
37894
38745
|
);
|
|
38746
|
+
output = taskResult.output;
|
|
38747
|
+
metadata = taskResult.metadata;
|
|
38748
|
+
tags = taskResult.tags;
|
|
37895
38749
|
if (tags.length) {
|
|
37896
38750
|
rootSpan.log({ output, metadata, expected, tags });
|
|
37897
38751
|
} else {
|
|
@@ -37901,20 +38755,18 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
37901
38755
|
await rootSpan.flush();
|
|
37902
38756
|
}
|
|
37903
38757
|
const scoringArgs = {
|
|
38758
|
+
id: datum.id,
|
|
37904
38759
|
input: datum.input,
|
|
37905
38760
|
expected: "expected" in datum ? datum.expected : void 0,
|
|
37906
38761
|
metadata,
|
|
37907
38762
|
output,
|
|
38763
|
+
tags,
|
|
37908
38764
|
trace
|
|
37909
38765
|
};
|
|
37910
38766
|
const { trace: _trace, ...scoringArgsForLogging } = scoringArgs;
|
|
37911
38767
|
const propagatedEvent = makeScorerPropagatedEvent(
|
|
37912
38768
|
await rootSpan.export()
|
|
37913
38769
|
);
|
|
37914
|
-
const getOtherFields = (s) => {
|
|
37915
|
-
const { metadata: _metadata, name: _name, ...rest } = s;
|
|
37916
|
-
return rest;
|
|
37917
|
-
};
|
|
37918
38770
|
const [scoreResults, classificationResults] = await Promise.all([
|
|
37919
38771
|
Promise.all(
|
|
37920
38772
|
(evaluator.scores ?? []).map(
|
|
@@ -37928,35 +38780,17 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
37928
38780
|
const scoreValue = await Promise.resolve(
|
|
37929
38781
|
score(scoringArgs)
|
|
37930
38782
|
);
|
|
37931
|
-
|
|
37932
|
-
|
|
37933
|
-
|
|
37934
|
-
if (!(typeof s === "object" && !isEmpty2(s))) {
|
|
37935
|
-
throw new Error(
|
|
37936
|
-
`When returning an array of scores, each score must be a non-empty object. Got: ${JSON.stringify(s)}`
|
|
37937
|
-
);
|
|
37938
|
-
}
|
|
37939
|
-
}
|
|
37940
|
-
}
|
|
37941
|
-
const results = Array.isArray(scoreValue) ? scoreValue : typeof scoreValue === "object" && !isEmpty2(scoreValue) ? [scoreValue] : [
|
|
37942
|
-
{
|
|
37943
|
-
name: scorerNames[score_idx],
|
|
37944
|
-
score: scoreValue
|
|
37945
|
-
}
|
|
37946
|
-
];
|
|
37947
|
-
const { resultMetadata, scoresRecord } = buildSpanScores(results);
|
|
37948
|
-
const resultOutput = results.length === 1 ? getOtherFields(results[0]) : results.reduce(
|
|
37949
|
-
(prev, s) => mergeDicts(prev, {
|
|
37950
|
-
[s.name]: getOtherFields(s)
|
|
37951
|
-
}),
|
|
37952
|
-
{}
|
|
38783
|
+
const prepared = _internalPrepareEvaluatorScore(
|
|
38784
|
+
scoreValue,
|
|
38785
|
+
scorerNames[score_idx]
|
|
37953
38786
|
);
|
|
38787
|
+
if (prepared.results === null) return null;
|
|
37954
38788
|
span.log({
|
|
37955
|
-
output:
|
|
37956
|
-
metadata:
|
|
37957
|
-
scores:
|
|
38789
|
+
output: prepared.output,
|
|
38790
|
+
metadata: prepared.metadata,
|
|
38791
|
+
scores: prepared.scores
|
|
37958
38792
|
});
|
|
37959
|
-
return results;
|
|
38793
|
+
return prepared.results;
|
|
37960
38794
|
}
|
|
37961
38795
|
)
|
|
37962
38796
|
)
|
|
@@ -37973,24 +38807,16 @@ async function runEvaluatorInternal(experiment, evaluator, progressReporter, fil
|
|
|
37973
38807
|
const classifierValue = await Promise.resolve(
|
|
37974
38808
|
classifier(scoringArgs)
|
|
37975
38809
|
);
|
|
37976
|
-
|
|
37977
|
-
|
|
37978
|
-
|
|
37979
|
-
result,
|
|
37980
|
-
classifierNames[idx]
|
|
37981
|
-
)
|
|
37982
|
-
);
|
|
37983
|
-
const resultOutput = rawResults.length === 1 ? toClassificationItem(rawResults[0]) : rawResults.reduce(
|
|
37984
|
-
(prev, r) => mergeDicts(prev, {
|
|
37985
|
-
[r.name]: toClassificationItem(r)
|
|
37986
|
-
}),
|
|
37987
|
-
{}
|
|
38810
|
+
const prepared = _internalPrepareEvaluatorClassification(
|
|
38811
|
+
classifierValue,
|
|
38812
|
+
classifierNames[idx]
|
|
37988
38813
|
);
|
|
38814
|
+
if (prepared.results === null) return null;
|
|
37989
38815
|
span.log({
|
|
37990
|
-
output:
|
|
37991
|
-
metadata:
|
|
38816
|
+
output: prepared.output,
|
|
38817
|
+
metadata: prepared.metadata
|
|
37992
38818
|
});
|
|
37993
|
-
return
|
|
38819
|
+
return prepared.results;
|
|
37994
38820
|
}
|
|
37995
38821
|
)
|
|
37996
38822
|
)
|
|
@@ -38219,7 +39045,7 @@ function accumulateScores(accumulator, scores) {
|
|
|
38219
39045
|
}
|
|
38220
39046
|
}
|
|
38221
39047
|
function ensureScoreAccumulator(results) {
|
|
38222
|
-
const accumulator =
|
|
39048
|
+
const accumulator = /* @__PURE__ */ Object.create(null);
|
|
38223
39049
|
for (const result of results) {
|
|
38224
39050
|
accumulateScores(accumulator, result.scores);
|
|
38225
39051
|
}
|
|
@@ -39393,6 +40219,8 @@ async function getDataset(state, data) {
|
|
|
39393
40219
|
environment: data.dataset_environment ?? void 0,
|
|
39394
40220
|
_internal_btql: data._internal_btql ?? void 0
|
|
39395
40221
|
});
|
|
40222
|
+
} else if ("experiment_name" in data) {
|
|
40223
|
+
return BaseExperiment({ name: data.experiment_name });
|
|
39396
40224
|
} else {
|
|
39397
40225
|
return data.data;
|
|
39398
40226
|
}
|