braintrust 3.1.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -0
- package/dev/dist/index.d.mts +187 -903
- package/dev/dist/index.d.ts +187 -903
- package/dev/dist/index.js +2942 -323
- package/dev/dist/index.mjs +2704 -85
- package/dist/auto-instrumentations/bundler/esbuild.cjs +401 -0
- package/dist/auto-instrumentations/bundler/esbuild.d.mts +8 -0
- package/dist/auto-instrumentations/bundler/esbuild.d.ts +8 -0
- package/dist/auto-instrumentations/bundler/esbuild.mjs +10 -0
- package/dist/auto-instrumentations/bundler/rollup.cjs +401 -0
- package/dist/auto-instrumentations/bundler/rollup.d.mts +8 -0
- package/dist/auto-instrumentations/bundler/rollup.d.ts +8 -0
- package/dist/auto-instrumentations/bundler/rollup.mjs +10 -0
- package/dist/auto-instrumentations/bundler/vite.cjs +401 -0
- package/dist/auto-instrumentations/bundler/vite.d.mts +8 -0
- package/dist/auto-instrumentations/bundler/vite.d.ts +8 -0
- package/dist/auto-instrumentations/bundler/vite.mjs +10 -0
- package/dist/auto-instrumentations/bundler/webpack.cjs +401 -0
- package/dist/auto-instrumentations/bundler/webpack.d.mts +8 -0
- package/dist/auto-instrumentations/bundler/webpack.d.ts +8 -0
- package/dist/auto-instrumentations/bundler/webpack.mjs +10 -0
- package/dist/auto-instrumentations/chunk-KVX7OFPD.mjs +288 -0
- package/dist/auto-instrumentations/chunk-OLOPGWTJ.mjs +89 -0
- package/dist/auto-instrumentations/chunk-XDBPUTVE.mjs +22 -0
- package/dist/auto-instrumentations/chunk-ZEC7BCL4.mjs +22 -0
- package/dist/auto-instrumentations/hook.mjs +378 -0
- package/dist/auto-instrumentations/index.cjs +318 -0
- package/dist/auto-instrumentations/index.d.mts +69 -0
- package/dist/auto-instrumentations/index.d.ts +69 -0
- package/dist/auto-instrumentations/index.mjs +14 -0
- package/dist/auto-instrumentations/loader/cjs-patch.cjs +116 -0
- package/dist/auto-instrumentations/loader/cjs-patch.d.mts +28 -0
- package/dist/auto-instrumentations/loader/cjs-patch.d.ts +28 -0
- package/dist/auto-instrumentations/loader/cjs-patch.mjs +66 -0
- package/dist/auto-instrumentations/loader/esm-hook.mjs +72 -0
- package/dist/auto-instrumentations/loader/get-package-version.cjs +46 -0
- package/dist/auto-instrumentations/loader/get-package-version.d.mts +7 -0
- package/dist/auto-instrumentations/loader/get-package-version.d.ts +7 -0
- package/dist/auto-instrumentations/loader/get-package-version.mjs +6 -0
- package/dist/auto-instrumentations/plugin-Df3qKIl2.d.mts +22 -0
- package/dist/auto-instrumentations/plugin-Df3qKIl2.d.ts +22 -0
- package/dist/browser.d.mts +484 -980
- package/dist/browser.d.ts +484 -980
- package/dist/browser.js +3437 -386
- package/dist/browser.mjs +3437 -386
- package/dist/cli.js +2890 -228
- package/dist/edge-light.d.mts +1 -1
- package/dist/edge-light.d.ts +1 -1
- package/dist/edge-light.js +3134 -86
- package/dist/edge-light.mjs +3134 -86
- package/dist/index.d.mts +484 -980
- package/dist/index.d.ts +484 -980
- package/dist/index.js +3754 -703
- package/dist/index.mjs +3615 -564
- package/dist/instrumentation/index.d.mts +323 -0
- package/dist/instrumentation/index.d.ts +323 -0
- package/dist/instrumentation/index.js +9942 -0
- package/dist/instrumentation/index.mjs +9902 -0
- package/dist/workerd.d.mts +1 -1
- package/dist/workerd.d.ts +1 -1
- package/dist/workerd.js +3134 -86
- package/dist/workerd.mjs +3134 -86
- package/package.json +46 -7
package/dist/workerd.js
CHANGED
|
@@ -82,6 +82,7 @@ __export(workerd_exports, {
|
|
|
82
82
|
addAzureBlobHeaders: () => addAzureBlobHeaders,
|
|
83
83
|
braintrustStreamChunkSchema: () => braintrustStreamChunkSchema,
|
|
84
84
|
buildLocalSummary: () => buildLocalSummary,
|
|
85
|
+
configureInstrumentation: () => configureInstrumentation,
|
|
85
86
|
constructLogs3OverflowRequest: () => constructLogs3OverflowRequest,
|
|
86
87
|
createFinalValuePassThroughStream: () => createFinalValuePassThroughStream,
|
|
87
88
|
currentExperiment: () => currentExperiment,
|
|
@@ -158,7 +159,8 @@ __export(workerd_exports, {
|
|
|
158
159
|
wrapMastraAgent: () => wrapMastraAgent,
|
|
159
160
|
wrapOpenAI: () => wrapOpenAI,
|
|
160
161
|
wrapOpenAIv4: () => wrapOpenAIv4,
|
|
161
|
-
wrapTraced: () => wrapTraced
|
|
162
|
+
wrapTraced: () => wrapTraced,
|
|
163
|
+
wrapVitest: () => wrapVitest
|
|
162
164
|
});
|
|
163
165
|
module.exports = __toCommonJS(workerd_exports);
|
|
164
166
|
|
|
@@ -1676,6 +1678,7 @@ var ApiKey = import_v36.z.object({
|
|
|
1676
1678
|
var TriggeredFunctionState = import_v36.z.object({
|
|
1677
1679
|
triggered_xact_id: import_v36.z.string(),
|
|
1678
1680
|
completed_xact_id: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
1681
|
+
idempotency_key: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
1679
1682
|
attempts: import_v36.z.number().int().gte(0).optional().default(0),
|
|
1680
1683
|
scope: import_v36.z.union([
|
|
1681
1684
|
import_v36.z.object({ type: import_v36.z.literal("span") }),
|
|
@@ -1708,7 +1711,8 @@ var AsyncScoringControl = import_v36.z.union([
|
|
|
1708
1711
|
scope: import_v36.z.union([
|
|
1709
1712
|
import_v36.z.object({ type: import_v36.z.literal("span") }),
|
|
1710
1713
|
import_v36.z.object({ type: import_v36.z.literal("trace") })
|
|
1711
|
-
])
|
|
1714
|
+
]),
|
|
1715
|
+
idempotency_key: import_v36.z.string().optional()
|
|
1712
1716
|
})
|
|
1713
1717
|
).min(1)
|
|
1714
1718
|
}),
|
|
@@ -1753,7 +1757,8 @@ var FunctionTypeEnum = import_v36.z.enum([
|
|
|
1753
1757
|
"facet",
|
|
1754
1758
|
"classifier",
|
|
1755
1759
|
"tag",
|
|
1756
|
-
"parameters"
|
|
1760
|
+
"parameters",
|
|
1761
|
+
"sandbox"
|
|
1757
1762
|
]);
|
|
1758
1763
|
var NullableSavedFunctionId = import_v36.z.union([
|
|
1759
1764
|
import_v36.z.object({
|
|
@@ -1768,66 +1773,14 @@ var NullableSavedFunctionId = import_v36.z.union([
|
|
|
1768
1773
|
}),
|
|
1769
1774
|
import_v36.z.null()
|
|
1770
1775
|
]);
|
|
1771
|
-
var TopicMapReport = import_v36.z.object({
|
|
1772
|
-
version: import_v36.z.literal(1),
|
|
1773
|
-
created_at: import_v36.z.string().optional(),
|
|
1774
|
-
settings: import_v36.z.object({
|
|
1775
|
-
algorithm: import_v36.z.enum(["hdbscan", "kmeans", "hierarchical"]),
|
|
1776
|
-
dimension_reduction: import_v36.z.enum(["umap", "pca", "none"]),
|
|
1777
|
-
vector_field: import_v36.z.string(),
|
|
1778
|
-
embedding_model: import_v36.z.string(),
|
|
1779
|
-
n_clusters: import_v36.z.union([import_v36.z.number(), import_v36.z.null()]).optional(),
|
|
1780
|
-
umap_dimensions: import_v36.z.union([import_v36.z.number(), import_v36.z.null()]).optional(),
|
|
1781
|
-
min_cluster_size: import_v36.z.union([import_v36.z.number(), import_v36.z.null()]).optional(),
|
|
1782
|
-
min_samples: import_v36.z.union([import_v36.z.number(), import_v36.z.null()]).optional()
|
|
1783
|
-
}),
|
|
1784
|
-
query_settings: import_v36.z.object({
|
|
1785
|
-
hierarchy_threshold: import_v36.z.union([import_v36.z.number(), import_v36.z.null()]),
|
|
1786
|
-
auto_naming: import_v36.z.boolean(),
|
|
1787
|
-
skip_cache: import_v36.z.boolean(),
|
|
1788
|
-
viz_mode: import_v36.z.enum(["bar", "scatter"]),
|
|
1789
|
-
naming_model: import_v36.z.string()
|
|
1790
|
-
}).partial(),
|
|
1791
|
-
clusters: import_v36.z.array(
|
|
1792
|
-
import_v36.z.object({
|
|
1793
|
-
cluster_id: import_v36.z.number(),
|
|
1794
|
-
parent_cluster_id: import_v36.z.union([import_v36.z.number(), import_v36.z.null()]).optional(),
|
|
1795
|
-
topic_id: import_v36.z.string(),
|
|
1796
|
-
count: import_v36.z.number(),
|
|
1797
|
-
sample_texts: import_v36.z.array(import_v36.z.string()),
|
|
1798
|
-
samples: import_v36.z.array(
|
|
1799
|
-
import_v36.z.object({
|
|
1800
|
-
id: import_v36.z.string(),
|
|
1801
|
-
text: import_v36.z.string(),
|
|
1802
|
-
root_span_id: import_v36.z.string(),
|
|
1803
|
-
span_id: import_v36.z.string()
|
|
1804
|
-
})
|
|
1805
|
-
),
|
|
1806
|
-
name: import_v36.z.string().optional(),
|
|
1807
|
-
description: import_v36.z.string().optional(),
|
|
1808
|
-
keywords: import_v36.z.array(import_v36.z.string()).optional(),
|
|
1809
|
-
centroid: import_v36.z.array(import_v36.z.number()).optional(),
|
|
1810
|
-
parent_id: import_v36.z.union([import_v36.z.number(), import_v36.z.null()]).optional(),
|
|
1811
|
-
is_leaf: import_v36.z.boolean().optional(),
|
|
1812
|
-
depth: import_v36.z.number().optional()
|
|
1813
|
-
})
|
|
1814
|
-
),
|
|
1815
|
-
embedding_points: import_v36.z.array(
|
|
1816
|
-
import_v36.z.object({
|
|
1817
|
-
x: import_v36.z.number(),
|
|
1818
|
-
y: import_v36.z.number(),
|
|
1819
|
-
cluster: import_v36.z.number(),
|
|
1820
|
-
text: import_v36.z.string().optional()
|
|
1821
|
-
})
|
|
1822
|
-
).optional()
|
|
1823
|
-
});
|
|
1824
1776
|
var TopicMapData = import_v36.z.object({
|
|
1825
1777
|
type: import_v36.z.literal("topic_map"),
|
|
1826
1778
|
source_facet: import_v36.z.string(),
|
|
1827
1779
|
embedding_model: import_v36.z.string(),
|
|
1828
|
-
bundle_key: import_v36.z.string(),
|
|
1829
|
-
|
|
1830
|
-
|
|
1780
|
+
bundle_key: import_v36.z.string().optional(),
|
|
1781
|
+
report_key: import_v36.z.string().optional(),
|
|
1782
|
+
topic_names: import_v36.z.record(import_v36.z.string()).optional(),
|
|
1783
|
+
distance_threshold: import_v36.z.number().optional()
|
|
1831
1784
|
});
|
|
1832
1785
|
var BatchedFacetData = import_v36.z.object({
|
|
1833
1786
|
type: import_v36.z.literal("batched_facet"),
|
|
@@ -1842,11 +1795,13 @@ var BatchedFacetData = import_v36.z.object({
|
|
|
1842
1795
|
})
|
|
1843
1796
|
),
|
|
1844
1797
|
topic_maps: import_v36.z.record(
|
|
1845
|
-
import_v36.z.
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1798
|
+
import_v36.z.array(
|
|
1799
|
+
import_v36.z.object({
|
|
1800
|
+
function_name: import_v36.z.string(),
|
|
1801
|
+
topic_map_id: import_v36.z.string().optional(),
|
|
1802
|
+
topic_map_data: TopicMapData
|
|
1803
|
+
})
|
|
1804
|
+
)
|
|
1850
1805
|
).optional()
|
|
1851
1806
|
});
|
|
1852
1807
|
var BraintrustModelParams = import_v36.z.object({
|
|
@@ -2025,9 +1980,20 @@ var CodeBundle = import_v36.z.object({
|
|
|
2025
1980
|
import_v36.z.object({ type: import_v36.z.literal("scorer"), index: import_v36.z.number().int().gte(0) })
|
|
2026
1981
|
])
|
|
2027
1982
|
}),
|
|
2028
|
-
import_v36.z.object({ type: import_v36.z.literal("function"), index: import_v36.z.number().int().gte(0) })
|
|
1983
|
+
import_v36.z.object({ type: import_v36.z.literal("function"), index: import_v36.z.number().int().gte(0) }),
|
|
1984
|
+
import_v36.z.object({
|
|
1985
|
+
type: import_v36.z.literal("sandbox"),
|
|
1986
|
+
sandbox_spec: import_v36.z.union([
|
|
1987
|
+
import_v36.z.object({ provider: import_v36.z.literal("modal"), snapshot_ref: import_v36.z.string() }),
|
|
1988
|
+
import_v36.z.object({ provider: import_v36.z.literal("lambda") })
|
|
1989
|
+
]),
|
|
1990
|
+
entrypoints: import_v36.z.array(import_v36.z.string()).optional(),
|
|
1991
|
+
eval_name: import_v36.z.string(),
|
|
1992
|
+
parameters: import_v36.z.object({}).partial().passthrough().optional(),
|
|
1993
|
+
evaluator_definition: import_v36.z.unknown().optional()
|
|
1994
|
+
})
|
|
2029
1995
|
]),
|
|
2030
|
-
bundle_id: import_v36.z.string(),
|
|
1996
|
+
bundle_id: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
2031
1997
|
preview: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional()
|
|
2032
1998
|
});
|
|
2033
1999
|
var Dataset = import_v36.z.object({
|
|
@@ -2129,7 +2095,7 @@ var EnvVar = import_v36.z.object({
|
|
|
2129
2095
|
used: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
2130
2096
|
metadata: import_v36.z.union([import_v36.z.object({}).partial().passthrough(), import_v36.z.null()]).optional(),
|
|
2131
2097
|
secret_type: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
2132
|
-
secret_category: import_v36.z.enum(["env_var", "ai_provider"]).optional().default("env_var")
|
|
2098
|
+
secret_category: import_v36.z.enum(["env_var", "ai_provider", "sandbox_provider"]).optional().default("env_var")
|
|
2133
2099
|
});
|
|
2134
2100
|
var EvalStatusPageTheme = import_v36.z.enum(["light", "dark"]);
|
|
2135
2101
|
var EvalStatusPageConfig = import_v36.z.object({
|
|
@@ -2431,7 +2397,8 @@ var FunctionTypeEnumNullish = import_v36.z.union([
|
|
|
2431
2397
|
"facet",
|
|
2432
2398
|
"classifier",
|
|
2433
2399
|
"tag",
|
|
2434
|
-
"parameters"
|
|
2400
|
+
"parameters",
|
|
2401
|
+
"sandbox"
|
|
2435
2402
|
]),
|
|
2436
2403
|
import_v36.z.null()
|
|
2437
2404
|
]);
|
|
@@ -2664,7 +2631,8 @@ var FunctionObjectType = import_v36.z.enum([
|
|
|
2664
2631
|
"preprocessor",
|
|
2665
2632
|
"facet",
|
|
2666
2633
|
"classifier",
|
|
2667
|
-
"parameters"
|
|
2634
|
+
"parameters",
|
|
2635
|
+
"sandbox"
|
|
2668
2636
|
]);
|
|
2669
2637
|
var FunctionOutputType = import_v36.z.enum([
|
|
2670
2638
|
"completion",
|
|
@@ -2800,6 +2768,7 @@ var Organization = import_v36.z.object({
|
|
|
2800
2768
|
name: import_v36.z.string(),
|
|
2801
2769
|
api_url: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
2802
2770
|
is_universal_api: import_v36.z.union([import_v36.z.boolean(), import_v36.z.null()]).optional(),
|
|
2771
|
+
is_dataplane_private: import_v36.z.union([import_v36.z.boolean(), import_v36.z.null()]).optional(),
|
|
2803
2772
|
proxy_url: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
2804
2773
|
realtime_url: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
2805
2774
|
created: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
@@ -2824,7 +2793,7 @@ var ProjectSettings = import_v36.z.union([
|
|
|
2824
2793
|
import_v36.z.array(
|
|
2825
2794
|
import_v36.z.object({
|
|
2826
2795
|
url: import_v36.z.string(),
|
|
2827
|
-
name: import_v36.z.string(),
|
|
2796
|
+
name: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional(),
|
|
2828
2797
|
description: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional()
|
|
2829
2798
|
})
|
|
2830
2799
|
),
|
|
@@ -2850,6 +2819,25 @@ var RetentionObjectType = import_v36.z.enum([
|
|
|
2850
2819
|
"experiment",
|
|
2851
2820
|
"dataset"
|
|
2852
2821
|
]);
|
|
2822
|
+
var TopicMapFunctionAutomation = import_v36.z.object({
|
|
2823
|
+
function: SavedFunctionId.and(import_v36.z.unknown()),
|
|
2824
|
+
btql_filter: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional()
|
|
2825
|
+
});
|
|
2826
|
+
var TopicAutomationDataScope = import_v36.z.union([
|
|
2827
|
+
import_v36.z.object({ type: import_v36.z.literal("project_logs") }),
|
|
2828
|
+
import_v36.z.object({ type: import_v36.z.literal("project_experiments") }),
|
|
2829
|
+
import_v36.z.object({ type: import_v36.z.literal("experiment"), experiment_id: import_v36.z.string() }),
|
|
2830
|
+
import_v36.z.null()
|
|
2831
|
+
]);
|
|
2832
|
+
var TopicAutomationConfig = import_v36.z.object({
|
|
2833
|
+
event_type: import_v36.z.literal("topic"),
|
|
2834
|
+
sampling_rate: import_v36.z.number().gte(0).lte(1),
|
|
2835
|
+
facet_functions: import_v36.z.array(SavedFunctionId),
|
|
2836
|
+
topic_map_functions: import_v36.z.array(TopicMapFunctionAutomation),
|
|
2837
|
+
scope: import_v36.z.union([SpanScope, TraceScope, GroupScope, import_v36.z.null()]).optional(),
|
|
2838
|
+
data_scope: TopicAutomationDataScope.optional(),
|
|
2839
|
+
btql_filter: import_v36.z.union([import_v36.z.string(), import_v36.z.null()]).optional()
|
|
2840
|
+
});
|
|
2853
2841
|
var ProjectAutomation = import_v36.z.object({
|
|
2854
2842
|
id: import_v36.z.string().uuid(),
|
|
2855
2843
|
project_id: import_v36.z.string().uuid(),
|
|
@@ -2906,7 +2894,8 @@ var ProjectAutomation = import_v36.z.object({
|
|
|
2906
2894
|
message_template: import_v36.z.string().optional()
|
|
2907
2895
|
})
|
|
2908
2896
|
])
|
|
2909
|
-
})
|
|
2897
|
+
}),
|
|
2898
|
+
TopicAutomationConfig
|
|
2910
2899
|
])
|
|
2911
2900
|
});
|
|
2912
2901
|
var ProjectLogsEvent = import_v36.z.object({
|
|
@@ -3096,6 +3085,8 @@ var RunEval = import_v36.z.object({
|
|
|
3096
3085
|
}),
|
|
3097
3086
|
import_v36.z.object({ data: import_v36.z.array(import_v36.z.unknown()) })
|
|
3098
3087
|
]),
|
|
3088
|
+
name: import_v36.z.string().optional(),
|
|
3089
|
+
parameters: import_v36.z.object({}).partial().passthrough().optional(),
|
|
3099
3090
|
task: FunctionId.and(import_v36.z.unknown()),
|
|
3100
3091
|
scores: import_v36.z.array(FunctionId),
|
|
3101
3092
|
experiment_name: import_v36.z.string().optional(),
|
|
@@ -4801,7 +4792,7 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
4801
4792
|
try {
|
|
4802
4793
|
const resp = await this.get("ping");
|
|
4803
4794
|
return resp.status === 200;
|
|
4804
|
-
} catch
|
|
4795
|
+
} catch {
|
|
4805
4796
|
return false;
|
|
4806
4797
|
}
|
|
4807
4798
|
}
|
|
@@ -5310,9 +5301,9 @@ function logFeedbackImpl(state, parentObjectType, parentObjectId, {
|
|
|
5310
5301
|
expected,
|
|
5311
5302
|
tags
|
|
5312
5303
|
});
|
|
5313
|
-
|
|
5314
|
-
updateEvent = Object.fromEntries(
|
|
5315
|
-
Object.entries(
|
|
5304
|
+
const { metadata, ...rawUpdateEvent } = deepCopyEvent(validatedEvent);
|
|
5305
|
+
const updateEvent = Object.fromEntries(
|
|
5306
|
+
Object.entries(rawUpdateEvent).filter(([_, v]) => !isEmpty2(v))
|
|
5316
5307
|
);
|
|
5317
5308
|
const parentIds = async () => new SpanComponentsV3({
|
|
5318
5309
|
object_type: parentObjectType,
|
|
@@ -7500,7 +7491,8 @@ function validateAndSanitizeExperimentLogPartialArgs(event) {
|
|
|
7500
7491
|
if (Array.isArray(event.scores)) {
|
|
7501
7492
|
throw new Error("scores must be an object, not an array");
|
|
7502
7493
|
}
|
|
7503
|
-
for (
|
|
7494
|
+
for (const [name, rawScore] of Object.entries(event.scores)) {
|
|
7495
|
+
let score = rawScore;
|
|
7504
7496
|
if (typeof name !== "string") {
|
|
7505
7497
|
throw new Error("score names must be strings");
|
|
7506
7498
|
}
|
|
@@ -7942,7 +7934,11 @@ var Experiment2 = class extends ObjectFetcher {
|
|
|
7942
7934
|
* @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
7943
7935
|
*/
|
|
7944
7936
|
async summarize(options = {}) {
|
|
7945
|
-
|
|
7937
|
+
const {
|
|
7938
|
+
summarizeScores = true,
|
|
7939
|
+
comparisonExperimentId: comparisonExperimentIdOpt
|
|
7940
|
+
} = options || {};
|
|
7941
|
+
let comparisonExperimentId = comparisonExperimentIdOpt;
|
|
7946
7942
|
const state = await this.getState();
|
|
7947
7943
|
const projectUrl = `${state.appPublicUrl}/app/${encodeURIComponent(
|
|
7948
7944
|
state.orgName
|
|
@@ -9126,8 +9122,7 @@ var RemoteEvalParameters = class {
|
|
|
9126
9122
|
return true;
|
|
9127
9123
|
}
|
|
9128
9124
|
static isParameters(x) {
|
|
9129
|
-
return typeof x === "object" && x !== null && "__braintrust_parameters_marker" in x &&
|
|
9130
|
-
x.__braintrust_parameters_marker === true;
|
|
9125
|
+
return typeof x === "object" && x !== null && "__braintrust_parameters_marker" in x && x.__braintrust_parameters_marker === true;
|
|
9131
9126
|
}
|
|
9132
9127
|
};
|
|
9133
9128
|
var TEST_API_KEY = "___TEST_API_KEY__THIS_IS_NOT_REAL___";
|
|
@@ -9310,6 +9305,7 @@ __export(exports_exports, {
|
|
|
9310
9305
|
addAzureBlobHeaders: () => addAzureBlobHeaders,
|
|
9311
9306
|
braintrustStreamChunkSchema: () => braintrustStreamChunkSchema,
|
|
9312
9307
|
buildLocalSummary: () => buildLocalSummary,
|
|
9308
|
+
configureInstrumentation: () => configureInstrumentation,
|
|
9313
9309
|
constructLogs3OverflowRequest: () => constructLogs3OverflowRequest,
|
|
9314
9310
|
createFinalValuePassThroughStream: () => createFinalValuePassThroughStream,
|
|
9315
9311
|
currentExperiment: () => currentExperiment,
|
|
@@ -9385,7 +9381,8 @@ __export(exports_exports, {
|
|
|
9385
9381
|
wrapMastraAgent: () => wrapMastraAgent,
|
|
9386
9382
|
wrapOpenAI: () => wrapOpenAI,
|
|
9387
9383
|
wrapOpenAIv4: () => wrapOpenAIv4,
|
|
9388
|
-
wrapTraced: () => wrapTraced
|
|
9384
|
+
wrapTraced: () => wrapTraced,
|
|
9385
|
+
wrapVitest: () => wrapVitest
|
|
9389
9386
|
});
|
|
9390
9387
|
|
|
9391
9388
|
// src/functions/invoke.ts
|
|
@@ -9541,7 +9538,7 @@ function parseEventFromResponseCreateResult(result) {
|
|
|
9541
9538
|
data.output = processImagesInOutput(result.output);
|
|
9542
9539
|
}
|
|
9543
9540
|
if (result) {
|
|
9544
|
-
const { output, usage, ...metadata } = result;
|
|
9541
|
+
const { output: _output, usage: _usage, ...metadata } = result;
|
|
9545
9542
|
if (Object.keys(metadata).length > 0) {
|
|
9546
9543
|
data.metadata = metadata;
|
|
9547
9544
|
}
|
|
@@ -9604,7 +9601,7 @@ function parseEventFromResponseParseResult(result) {
|
|
|
9604
9601
|
data.output = processImagesInOutput(result.output);
|
|
9605
9602
|
}
|
|
9606
9603
|
if (result) {
|
|
9607
|
-
const { output, usage, ...metadata } = result;
|
|
9604
|
+
const { output: _output, usage: _usage, ...metadata } = result;
|
|
9608
9605
|
if (Object.keys(metadata).length > 0) {
|
|
9609
9606
|
data.metadata = metadata;
|
|
9610
9607
|
}
|
|
@@ -9648,7 +9645,7 @@ function parseLogFromItem(item) {
|
|
|
9648
9645
|
data.output = processImagesInOutput(response.output);
|
|
9649
9646
|
}
|
|
9650
9647
|
if (response) {
|
|
9651
|
-
const { usage, output, ...metadata } = response;
|
|
9648
|
+
const { usage: _usage, output: _output, ...metadata } = response;
|
|
9652
9649
|
if (Object.keys(metadata).length > 0) {
|
|
9653
9650
|
data.metadata = metadata;
|
|
9654
9651
|
}
|
|
@@ -10039,7 +10036,7 @@ function wrapChatCompletion(completion) {
|
|
|
10039
10036
|
);
|
|
10040
10037
|
const { data: ret, response } = await completionResponse.withResponse();
|
|
10041
10038
|
logHeaders(response, span);
|
|
10042
|
-
const { messages, ...rest } = params;
|
|
10039
|
+
const { messages: _messages, ...rest } = params;
|
|
10043
10040
|
span.log({
|
|
10044
10041
|
metadata: {
|
|
10045
10042
|
...rest
|
|
@@ -10362,7 +10359,7 @@ function convertDataToBlob(data, mediaType) {
|
|
|
10362
10359
|
} else if (typeof Buffer !== "undefined" && data instanceof Buffer) {
|
|
10363
10360
|
return new Blob([data], { type: mediaType });
|
|
10364
10361
|
}
|
|
10365
|
-
} catch
|
|
10362
|
+
} catch {
|
|
10366
10363
|
return null;
|
|
10367
10364
|
}
|
|
10368
10365
|
return null;
|
|
@@ -12635,7 +12632,6 @@ function apiPromiseProxy2(apiPromise, span, onThen) {
|
|
|
12635
12632
|
return function(onFulfilled, onRejected) {
|
|
12636
12633
|
return thenFunc.call(
|
|
12637
12634
|
target,
|
|
12638
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
12639
12635
|
async (result) => {
|
|
12640
12636
|
try {
|
|
12641
12637
|
const processed = onThen(result);
|
|
@@ -13847,6 +13843,582 @@ function tryToDict(obj) {
|
|
|
13847
13843
|
return null;
|
|
13848
13844
|
}
|
|
13849
13845
|
|
|
13846
|
+
// src/wrappers/vitest/context-manager.ts
|
|
13847
|
+
var VitestContextManager = class {
|
|
13848
|
+
/**
|
|
13849
|
+
* AsyncLocalStorage for experiment context isolation.
|
|
13850
|
+
* Each async execution flow (test, concurrent test, worker thread) gets its own context.
|
|
13851
|
+
*/
|
|
13852
|
+
contextStorage;
|
|
13853
|
+
constructor() {
|
|
13854
|
+
this.contextStorage = isomorph_default.newAsyncLocalStorage();
|
|
13855
|
+
}
|
|
13856
|
+
getCurrentContext() {
|
|
13857
|
+
return this.contextStorage.getStore();
|
|
13858
|
+
}
|
|
13859
|
+
setContext(context) {
|
|
13860
|
+
this.contextStorage.enterWith(context);
|
|
13861
|
+
}
|
|
13862
|
+
runInContext(context, callback) {
|
|
13863
|
+
return this.contextStorage.run(context, callback);
|
|
13864
|
+
}
|
|
13865
|
+
createChildContext(dataset, experiment) {
|
|
13866
|
+
const parent = this.getCurrentContext();
|
|
13867
|
+
return {
|
|
13868
|
+
dataset,
|
|
13869
|
+
experiment,
|
|
13870
|
+
datasetExamples: /* @__PURE__ */ new Map(),
|
|
13871
|
+
parent,
|
|
13872
|
+
flushResolved: true,
|
|
13873
|
+
passed: 0,
|
|
13874
|
+
failed: 0
|
|
13875
|
+
};
|
|
13876
|
+
}
|
|
13877
|
+
};
|
|
13878
|
+
var _contextManager;
|
|
13879
|
+
function getVitestContextManager() {
|
|
13880
|
+
if (!_contextManager) {
|
|
13881
|
+
_contextManager = new VitestContextManager();
|
|
13882
|
+
}
|
|
13883
|
+
return _contextManager;
|
|
13884
|
+
}
|
|
13885
|
+
|
|
13886
|
+
// src/wrappers/vitest/flush-manager.ts
|
|
13887
|
+
var FlushCoordinator = class {
|
|
13888
|
+
activeFlushes = /* @__PURE__ */ new Map();
|
|
13889
|
+
async coordinateFlush(context, config) {
|
|
13890
|
+
if (!context) return;
|
|
13891
|
+
const experimentId = await context.experiment.id;
|
|
13892
|
+
if (this.activeFlushes.has(experimentId)) {
|
|
13893
|
+
return this.activeFlushes.get(experimentId);
|
|
13894
|
+
}
|
|
13895
|
+
const flushPromise = this.doFlush(context, config);
|
|
13896
|
+
this.activeFlushes.set(experimentId, flushPromise);
|
|
13897
|
+
try {
|
|
13898
|
+
await flushPromise;
|
|
13899
|
+
} finally {
|
|
13900
|
+
this.activeFlushes.delete(experimentId);
|
|
13901
|
+
}
|
|
13902
|
+
}
|
|
13903
|
+
async doFlush(context, config) {
|
|
13904
|
+
let summary;
|
|
13905
|
+
try {
|
|
13906
|
+
summary = await context.experiment.summarize();
|
|
13907
|
+
} catch (error) {
|
|
13908
|
+
console.warn("Failed to generate experiment summary:", error);
|
|
13909
|
+
}
|
|
13910
|
+
try {
|
|
13911
|
+
await context.experiment.flush();
|
|
13912
|
+
} catch (error) {
|
|
13913
|
+
console.warn("Failed to flush experiment:", error);
|
|
13914
|
+
throw error;
|
|
13915
|
+
}
|
|
13916
|
+
if (summary && (config.displaySummary ?? true)) {
|
|
13917
|
+
console.log(formatExperimentSummary(summary));
|
|
13918
|
+
}
|
|
13919
|
+
}
|
|
13920
|
+
};
|
|
13921
|
+
var flushCoordinator = new FlushCoordinator();
|
|
13922
|
+
async function flushExperimentWithSync(context, config) {
|
|
13923
|
+
return flushCoordinator.coordinateFlush(context, config);
|
|
13924
|
+
}
|
|
13925
|
+
|
|
13926
|
+
// src/wrappers/vitest/scorers.ts
|
|
13927
|
+
async function runScorers(args) {
|
|
13928
|
+
const { scorers, output, expected, input, metadata, span } = args;
|
|
13929
|
+
const scorerArgs = {
|
|
13930
|
+
output,
|
|
13931
|
+
expected,
|
|
13932
|
+
input,
|
|
13933
|
+
metadata: metadata || {}
|
|
13934
|
+
};
|
|
13935
|
+
for (const scorer of scorers) {
|
|
13936
|
+
try {
|
|
13937
|
+
const result = await scorer(scorerArgs);
|
|
13938
|
+
const scores = normalizeScores(result);
|
|
13939
|
+
for (const score of scores) {
|
|
13940
|
+
if (score.metadata && Object.keys(score.metadata).length > 0) {
|
|
13941
|
+
span.log({
|
|
13942
|
+
scores: { [score.name]: score.score },
|
|
13943
|
+
metadata: score.metadata
|
|
13944
|
+
});
|
|
13945
|
+
} else {
|
|
13946
|
+
span.log({
|
|
13947
|
+
scores: { [score.name]: score.score }
|
|
13948
|
+
});
|
|
13949
|
+
}
|
|
13950
|
+
}
|
|
13951
|
+
} catch (scorerError) {
|
|
13952
|
+
console.warn("Braintrust: Scorer failed:", scorerError);
|
|
13953
|
+
span.log({
|
|
13954
|
+
metadata: {
|
|
13955
|
+
scorer_error: scorerError instanceof Error ? { message: scorerError.message, name: scorerError.name } : String(scorerError)
|
|
13956
|
+
}
|
|
13957
|
+
});
|
|
13958
|
+
}
|
|
13959
|
+
}
|
|
13960
|
+
}
|
|
13961
|
+
function isScore(val) {
|
|
13962
|
+
return "name" in val && "score" in val;
|
|
13963
|
+
}
|
|
13964
|
+
function normalizeScores(result) {
|
|
13965
|
+
if (result === null || result === void 0) {
|
|
13966
|
+
return [];
|
|
13967
|
+
}
|
|
13968
|
+
if (typeof result === "number") {
|
|
13969
|
+
return [{ name: "score", score: result }];
|
|
13970
|
+
}
|
|
13971
|
+
if (Array.isArray(result)) {
|
|
13972
|
+
return result.filter((s) => s !== null && s !== void 0);
|
|
13973
|
+
}
|
|
13974
|
+
if (typeof result === "object" && result !== null && isScore(result)) {
|
|
13975
|
+
return [result];
|
|
13976
|
+
}
|
|
13977
|
+
return [];
|
|
13978
|
+
}
|
|
13979
|
+
|
|
13980
|
+
// src/wrappers/vitest/wrapper.ts
|
|
13981
|
+
function formatExperimentSummary(summary) {
|
|
13982
|
+
const lines = [];
|
|
13983
|
+
lines.push("\n\u250C\u2500 Braintrust Experiment Summary \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510");
|
|
13984
|
+
lines.push(`\u2502 Experiment: ${summary.experimentName}`);
|
|
13985
|
+
if (Object.keys(summary.scores).length > 0) {
|
|
13986
|
+
lines.push("\u2502");
|
|
13987
|
+
lines.push("\u2502 Scores:");
|
|
13988
|
+
for (const [name, score] of Object.entries(summary.scores)) {
|
|
13989
|
+
const percent = (score.score * 100).toFixed(2);
|
|
13990
|
+
lines.push(`\u2502 ${name}: ${percent}%`);
|
|
13991
|
+
}
|
|
13992
|
+
}
|
|
13993
|
+
if (summary.metrics && Object.keys(summary.metrics).length > 0) {
|
|
13994
|
+
lines.push("\u2502");
|
|
13995
|
+
lines.push("\u2502 Metrics:");
|
|
13996
|
+
for (const [name, metric] of Object.entries(summary.metrics)) {
|
|
13997
|
+
const value = Number.isInteger(metric.metric) ? metric.metric.toFixed(0) : metric.metric.toFixed(2);
|
|
13998
|
+
const formatted = metric.unit === "$" ? `${metric.unit}${value}` : `${value}${metric.unit}`;
|
|
13999
|
+
lines.push(`\u2502 ${name}: ${formatted}`);
|
|
14000
|
+
}
|
|
14001
|
+
}
|
|
14002
|
+
if (summary.experimentUrl) {
|
|
14003
|
+
lines.push("\u2502");
|
|
14004
|
+
lines.push(`\u2502 View results: ${summary.experimentUrl}`);
|
|
14005
|
+
}
|
|
14006
|
+
lines.push("\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n");
|
|
14007
|
+
return lines.join("\n");
|
|
14008
|
+
}
|
|
14009
|
+
function getExperimentContext() {
|
|
14010
|
+
return getVitestContextManager().getCurrentContext() ?? null;
|
|
14011
|
+
}
|
|
14012
|
+
function wrapTest(originalTest, config) {
|
|
14013
|
+
const wrapBare = (testFn) => {
|
|
14014
|
+
const wrapped = function(name, configOrFn, maybeFn) {
|
|
14015
|
+
const isEnhanced = typeof configOrFn !== "function";
|
|
14016
|
+
const testConfig = isEnhanced ? configOrFn : void 0;
|
|
14017
|
+
if (isEnhanced && testConfig?.data && Array.isArray(testConfig.data)) {
|
|
14018
|
+
const dataRecords = testConfig.data;
|
|
14019
|
+
const testFn2 = maybeFn;
|
|
14020
|
+
if (!testFn2) {
|
|
14021
|
+
throw new Error(
|
|
14022
|
+
"Braintrust: test function required when using data array"
|
|
14023
|
+
);
|
|
14024
|
+
}
|
|
14025
|
+
dataRecords.forEach((record, index) => {
|
|
14026
|
+
const mergedConfig = {
|
|
14027
|
+
...testConfig,
|
|
14028
|
+
input: record.input,
|
|
14029
|
+
expected: record.expected,
|
|
14030
|
+
metadata: { ...testConfig.metadata, ...record.metadata },
|
|
14031
|
+
tags: [
|
|
14032
|
+
...testConfig.tags || [],
|
|
14033
|
+
...record.tags || []
|
|
14034
|
+
],
|
|
14035
|
+
data: void 0
|
|
14036
|
+
};
|
|
14037
|
+
wrappedTest(`${name} [${index}]`, mergedConfig, testFn2);
|
|
14038
|
+
});
|
|
14039
|
+
return;
|
|
14040
|
+
}
|
|
14041
|
+
let vitestOptions;
|
|
14042
|
+
if (testConfig) {
|
|
14043
|
+
const {
|
|
14044
|
+
input: _input,
|
|
14045
|
+
expected: _expected,
|
|
14046
|
+
metadata: _metadata,
|
|
14047
|
+
tags: _tags,
|
|
14048
|
+
scorers: _scorers,
|
|
14049
|
+
data: _data,
|
|
14050
|
+
...rest
|
|
14051
|
+
} = testConfig;
|
|
14052
|
+
vitestOptions = rest;
|
|
14053
|
+
}
|
|
14054
|
+
const hasVitestOptions = vitestOptions && Object.keys(vitestOptions).length > 0;
|
|
14055
|
+
const registrationContext = getExperimentContext();
|
|
14056
|
+
const testImplementation = async (vitestContext) => {
|
|
14057
|
+
const experimentContext = getExperimentContext() ?? registrationContext;
|
|
14058
|
+
const experiment = experimentContext?.experiment;
|
|
14059
|
+
if (config.onProgress) {
|
|
14060
|
+
config.onProgress({ type: "test_start", testName: name });
|
|
14061
|
+
}
|
|
14062
|
+
const startTime = performance.now();
|
|
14063
|
+
let passed = false;
|
|
14064
|
+
try {
|
|
14065
|
+
if (!experiment) {
|
|
14066
|
+
if (testConfig && maybeFn) {
|
|
14067
|
+
const params = {
|
|
14068
|
+
input: testConfig.input,
|
|
14069
|
+
expected: testConfig.expected,
|
|
14070
|
+
metadata: testConfig.metadata
|
|
14071
|
+
};
|
|
14072
|
+
const context = {
|
|
14073
|
+
...vitestContext,
|
|
14074
|
+
...params
|
|
14075
|
+
};
|
|
14076
|
+
const result2 = await maybeFn(context);
|
|
14077
|
+
passed = true;
|
|
14078
|
+
return result2;
|
|
14079
|
+
} else if (typeof configOrFn === "function") {
|
|
14080
|
+
const result2 = await configOrFn(vitestContext);
|
|
14081
|
+
passed = true;
|
|
14082
|
+
return result2;
|
|
14083
|
+
}
|
|
14084
|
+
passed = true;
|
|
14085
|
+
return;
|
|
14086
|
+
}
|
|
14087
|
+
const result = await experiment.traced(
|
|
14088
|
+
async (span) => {
|
|
14089
|
+
let testResult;
|
|
14090
|
+
try {
|
|
14091
|
+
if (testConfig && maybeFn) {
|
|
14092
|
+
const params = {
|
|
14093
|
+
input: testConfig.input,
|
|
14094
|
+
expected: testConfig.expected,
|
|
14095
|
+
metadata: testConfig.metadata
|
|
14096
|
+
};
|
|
14097
|
+
const context = {
|
|
14098
|
+
...vitestContext,
|
|
14099
|
+
...params
|
|
14100
|
+
};
|
|
14101
|
+
testResult = await maybeFn(context);
|
|
14102
|
+
} else if (typeof configOrFn === "function") {
|
|
14103
|
+
testResult = await configOrFn(vitestContext);
|
|
14104
|
+
}
|
|
14105
|
+
if (testConfig?.scorers && testConfig.scorers.length > 0) {
|
|
14106
|
+
await runScorers({
|
|
14107
|
+
scorers: testConfig.scorers,
|
|
14108
|
+
output: testResult,
|
|
14109
|
+
expected: testConfig.expected,
|
|
14110
|
+
input: testConfig.input,
|
|
14111
|
+
metadata: testConfig.metadata,
|
|
14112
|
+
span
|
|
14113
|
+
});
|
|
14114
|
+
}
|
|
14115
|
+
span.log({
|
|
14116
|
+
scores: {
|
|
14117
|
+
pass: 1
|
|
14118
|
+
}
|
|
14119
|
+
});
|
|
14120
|
+
if (testResult !== void 0) {
|
|
14121
|
+
span.log({
|
|
14122
|
+
output: testResult
|
|
14123
|
+
});
|
|
14124
|
+
}
|
|
14125
|
+
} catch (error) {
|
|
14126
|
+
if (testConfig?.scorers && testConfig.scorers.length > 0) {
|
|
14127
|
+
await runScorers({
|
|
14128
|
+
scorers: testConfig.scorers,
|
|
14129
|
+
output: testResult,
|
|
14130
|
+
expected: testConfig.expected,
|
|
14131
|
+
input: testConfig.input,
|
|
14132
|
+
metadata: testConfig.metadata,
|
|
14133
|
+
span
|
|
14134
|
+
});
|
|
14135
|
+
}
|
|
14136
|
+
span.log({
|
|
14137
|
+
scores: {
|
|
14138
|
+
pass: 0
|
|
14139
|
+
},
|
|
14140
|
+
metadata: {
|
|
14141
|
+
error: error instanceof Error ? {
|
|
14142
|
+
message: error.message,
|
|
14143
|
+
name: error.name,
|
|
14144
|
+
stack: error.stack
|
|
14145
|
+
} : String(error)
|
|
14146
|
+
}
|
|
14147
|
+
});
|
|
14148
|
+
throw error;
|
|
14149
|
+
}
|
|
14150
|
+
return testResult;
|
|
14151
|
+
},
|
|
14152
|
+
{
|
|
14153
|
+
name,
|
|
14154
|
+
spanAttributes: {
|
|
14155
|
+
type: "task" /* TASK */
|
|
14156
|
+
},
|
|
14157
|
+
event: testConfig ? {
|
|
14158
|
+
input: testConfig.input,
|
|
14159
|
+
expected: testConfig.expected,
|
|
14160
|
+
metadata: testConfig.metadata,
|
|
14161
|
+
tags: testConfig.tags
|
|
14162
|
+
} : void 0
|
|
14163
|
+
}
|
|
14164
|
+
);
|
|
14165
|
+
passed = true;
|
|
14166
|
+
return result;
|
|
14167
|
+
} catch (error) {
|
|
14168
|
+
passed = false;
|
|
14169
|
+
throw error;
|
|
14170
|
+
} finally {
|
|
14171
|
+
const duration = performance.now() - startTime;
|
|
14172
|
+
if (experimentContext) {
|
|
14173
|
+
if (passed) {
|
|
14174
|
+
experimentContext.passed = (experimentContext.passed ?? 0) + 1;
|
|
14175
|
+
} else {
|
|
14176
|
+
experimentContext.failed = (experimentContext.failed ?? 0) + 1;
|
|
14177
|
+
}
|
|
14178
|
+
}
|
|
14179
|
+
if (config.onProgress) {
|
|
14180
|
+
config.onProgress({
|
|
14181
|
+
type: "test_complete",
|
|
14182
|
+
testName: name,
|
|
14183
|
+
passed,
|
|
14184
|
+
duration
|
|
14185
|
+
});
|
|
14186
|
+
}
|
|
14187
|
+
}
|
|
14188
|
+
};
|
|
14189
|
+
return testFn(
|
|
14190
|
+
name,
|
|
14191
|
+
hasVitestOptions ? vitestOptions : void 0,
|
|
14192
|
+
testImplementation
|
|
14193
|
+
);
|
|
14194
|
+
};
|
|
14195
|
+
return wrapped;
|
|
14196
|
+
};
|
|
14197
|
+
const wrappedTest = wrapBare(originalTest);
|
|
14198
|
+
wrappedTest.skip = wrapBare(originalTest.skip);
|
|
14199
|
+
wrappedTest.only = wrapBare(originalTest.only);
|
|
14200
|
+
wrappedTest.concurrent = wrapBare(originalTest.concurrent);
|
|
14201
|
+
if (originalTest.todo) wrappedTest.todo = originalTest.todo;
|
|
14202
|
+
if (originalTest.each) wrappedTest.each = originalTest.each;
|
|
14203
|
+
return wrappedTest;
|
|
14204
|
+
}
|
|
14205
|
+
function wrapDescribe(originalDescribe, config, afterAll) {
|
|
14206
|
+
const wrapBare = (describeFn) => {
|
|
14207
|
+
const wrapped = function(suiteName, factory) {
|
|
14208
|
+
return describeFn(suiteName, () => {
|
|
14209
|
+
const contextManager = getVitestContextManager();
|
|
14210
|
+
let context = null;
|
|
14211
|
+
const getOrCreateContext = () => {
|
|
14212
|
+
if (!context) {
|
|
14213
|
+
const projectName = config.projectName || suiteName;
|
|
14214
|
+
const experiment = initExperiment(projectName, {
|
|
14215
|
+
experiment: `${suiteName}-${(/* @__PURE__ */ new Date()).toISOString()}`
|
|
14216
|
+
});
|
|
14217
|
+
context = contextManager.createChildContext(void 0, experiment);
|
|
14218
|
+
}
|
|
14219
|
+
return context;
|
|
14220
|
+
};
|
|
14221
|
+
const lazyContext = {
|
|
14222
|
+
get dataset() {
|
|
14223
|
+
return getOrCreateContext().dataset;
|
|
14224
|
+
},
|
|
14225
|
+
get experiment() {
|
|
14226
|
+
return getOrCreateContext().experiment;
|
|
14227
|
+
},
|
|
14228
|
+
get datasetExamples() {
|
|
14229
|
+
return getOrCreateContext().datasetExamples;
|
|
14230
|
+
},
|
|
14231
|
+
get parent() {
|
|
14232
|
+
return getOrCreateContext().parent;
|
|
14233
|
+
},
|
|
14234
|
+
get flushPromise() {
|
|
14235
|
+
return getOrCreateContext().flushPromise;
|
|
14236
|
+
},
|
|
14237
|
+
set flushPromise(value) {
|
|
14238
|
+
if (context) context.flushPromise = value;
|
|
14239
|
+
},
|
|
14240
|
+
get flushResolved() {
|
|
14241
|
+
return getOrCreateContext().flushResolved;
|
|
14242
|
+
},
|
|
14243
|
+
set flushResolved(value) {
|
|
14244
|
+
if (context) context.flushResolved = value;
|
|
14245
|
+
}
|
|
14246
|
+
};
|
|
14247
|
+
if (config.onProgress) {
|
|
14248
|
+
config.onProgress({ type: "suite_start", suiteName });
|
|
14249
|
+
}
|
|
14250
|
+
contextManager.setContext(lazyContext);
|
|
14251
|
+
factory();
|
|
14252
|
+
if (afterAll && (config.displaySummary ?? true)) {
|
|
14253
|
+
afterAll(async () => {
|
|
14254
|
+
await flushExperimentWithSync(context, config);
|
|
14255
|
+
if (config.onProgress) {
|
|
14256
|
+
config.onProgress({
|
|
14257
|
+
type: "suite_complete",
|
|
14258
|
+
suiteName,
|
|
14259
|
+
passed: context?.passed ?? 0,
|
|
14260
|
+
failed: context?.failed ?? 0
|
|
14261
|
+
});
|
|
14262
|
+
}
|
|
14263
|
+
});
|
|
14264
|
+
}
|
|
14265
|
+
});
|
|
14266
|
+
};
|
|
14267
|
+
return wrapped;
|
|
14268
|
+
};
|
|
14269
|
+
const wrappedDescribe = wrapBare(originalDescribe);
|
|
14270
|
+
wrappedDescribe.skip = wrapBare(originalDescribe.skip);
|
|
14271
|
+
wrappedDescribe.only = wrapBare(originalDescribe.only);
|
|
14272
|
+
wrappedDescribe.concurrent = wrapBare(originalDescribe.concurrent);
|
|
14273
|
+
if (originalDescribe.todo) wrappedDescribe.todo = originalDescribe.todo;
|
|
14274
|
+
if (originalDescribe.each)
|
|
14275
|
+
wrappedDescribe.each = originalDescribe.each;
|
|
14276
|
+
return wrappedDescribe;
|
|
14277
|
+
}
|
|
14278
|
+
|
|
14279
|
+
// src/wrappers/vitest/expect-wrapper.ts
|
|
14280
|
+
function proxyAssertion(assertion, value, key, span) {
|
|
14281
|
+
return new Proxy(assertion, {
|
|
14282
|
+
get(target, prop, receiver) {
|
|
14283
|
+
const original = Reflect.get(target, prop, receiver);
|
|
14284
|
+
if (typeof original === "function") {
|
|
14285
|
+
return function(...args) {
|
|
14286
|
+
let result;
|
|
14287
|
+
try {
|
|
14288
|
+
result = original.apply(target, args);
|
|
14289
|
+
} catch (err) {
|
|
14290
|
+
span.log({ output: { [key]: value }, scores: { [key]: 0 } });
|
|
14291
|
+
throw err;
|
|
14292
|
+
}
|
|
14293
|
+
if (result !== null && typeof result === "object" && "then" in result && typeof Reflect.get(result, "then") === "function") {
|
|
14294
|
+
return Promise.resolve(result).then(
|
|
14295
|
+
(v) => {
|
|
14296
|
+
span.log({ output: { [key]: value }, scores: { [key]: 1 } });
|
|
14297
|
+
return v;
|
|
14298
|
+
},
|
|
14299
|
+
(err) => {
|
|
14300
|
+
span.log({ output: { [key]: value }, scores: { [key]: 0 } });
|
|
14301
|
+
throw err;
|
|
14302
|
+
}
|
|
14303
|
+
);
|
|
14304
|
+
}
|
|
14305
|
+
span.log({ output: { [key]: value }, scores: { [key]: 1 } });
|
|
14306
|
+
return result;
|
|
14307
|
+
};
|
|
14308
|
+
}
|
|
14309
|
+
if (original !== null && typeof original === "object") {
|
|
14310
|
+
return proxyAssertion(original, value, key, span);
|
|
14311
|
+
}
|
|
14312
|
+
return original;
|
|
14313
|
+
}
|
|
14314
|
+
});
|
|
14315
|
+
}
|
|
14316
|
+
function wrapExpect(originalExpect) {
|
|
14317
|
+
const wrapped = function(value, message) {
|
|
14318
|
+
if (message === void 0) {
|
|
14319
|
+
return originalExpect(value);
|
|
14320
|
+
}
|
|
14321
|
+
const assertion = originalExpect(value, message);
|
|
14322
|
+
const span = currentSpan();
|
|
14323
|
+
if (!span) {
|
|
14324
|
+
return assertion;
|
|
14325
|
+
}
|
|
14326
|
+
if (assertion === null || typeof assertion !== "object") return assertion;
|
|
14327
|
+
return proxyAssertion(assertion, value, message, span);
|
|
14328
|
+
};
|
|
14329
|
+
return Object.assign(wrapped, originalExpect);
|
|
14330
|
+
}
|
|
14331
|
+
|
|
14332
|
+
// src/wrappers/vitest/index.ts
|
|
14333
|
+
function wrapVitest(vitestMethods, config = {}) {
|
|
14334
|
+
if (!vitestMethods.test) {
|
|
14335
|
+
throw new Error(
|
|
14336
|
+
"Braintrust: vitestMethods.test is required. Please pass in the test function from vitest."
|
|
14337
|
+
);
|
|
14338
|
+
}
|
|
14339
|
+
if (!vitestMethods.describe) {
|
|
14340
|
+
throw new Error(
|
|
14341
|
+
"Braintrust: vitestMethods.describe is required. Please pass in the describe function from vitest."
|
|
14342
|
+
);
|
|
14343
|
+
}
|
|
14344
|
+
if (!vitestMethods.expect) {
|
|
14345
|
+
throw new Error(
|
|
14346
|
+
"Braintrust: vitestMethods.expect is required. Please pass in the expect function from vitest."
|
|
14347
|
+
);
|
|
14348
|
+
}
|
|
14349
|
+
const wrappedTest = wrapTest(vitestMethods.test, config);
|
|
14350
|
+
const wrappedDescribe = wrapDescribe(
|
|
14351
|
+
vitestMethods.describe,
|
|
14352
|
+
config,
|
|
14353
|
+
vitestMethods.afterAll
|
|
14354
|
+
);
|
|
14355
|
+
return {
|
|
14356
|
+
test: wrappedTest,
|
|
14357
|
+
it: wrappedTest,
|
|
14358
|
+
expect: wrapExpect(vitestMethods.expect),
|
|
14359
|
+
describe: wrappedDescribe,
|
|
14360
|
+
beforeAll: vitestMethods.beforeAll || (() => {
|
|
14361
|
+
}),
|
|
14362
|
+
afterAll: vitestMethods.afterAll || (() => {
|
|
14363
|
+
}),
|
|
14364
|
+
beforeEach: vitestMethods.beforeEach,
|
|
14365
|
+
afterEach: vitestMethods.afterEach,
|
|
14366
|
+
logOutputs: (outputs) => {
|
|
14367
|
+
const span = currentSpan();
|
|
14368
|
+
if (!span) {
|
|
14369
|
+
console.warn(
|
|
14370
|
+
"Braintrust: No active span. logOutputs() must be called within a wrapped test."
|
|
14371
|
+
);
|
|
14372
|
+
return;
|
|
14373
|
+
}
|
|
14374
|
+
span.log({ output: outputs });
|
|
14375
|
+
},
|
|
14376
|
+
logFeedback: (feedback) => {
|
|
14377
|
+
const span = currentSpan();
|
|
14378
|
+
if (!span) {
|
|
14379
|
+
console.warn(
|
|
14380
|
+
"Braintrust: No active span. logFeedback() must be called within a wrapped test."
|
|
14381
|
+
);
|
|
14382
|
+
return;
|
|
14383
|
+
}
|
|
14384
|
+
span.log({
|
|
14385
|
+
scores: {
|
|
14386
|
+
[feedback.name]: feedback.score
|
|
14387
|
+
},
|
|
14388
|
+
metadata: feedback.metadata
|
|
14389
|
+
});
|
|
14390
|
+
},
|
|
14391
|
+
getCurrentSpan: () => {
|
|
14392
|
+
return currentSpan();
|
|
14393
|
+
},
|
|
14394
|
+
flushExperiment: async (options) => {
|
|
14395
|
+
const ctx = getExperimentContext();
|
|
14396
|
+
if (!ctx) {
|
|
14397
|
+
console.warn(
|
|
14398
|
+
"Braintrust: No experiment context found. Make sure you're using bt.describe() and calling flushExperiment() within an afterAll() hook."
|
|
14399
|
+
);
|
|
14400
|
+
return;
|
|
14401
|
+
}
|
|
14402
|
+
const shouldDisplaySummary = options?.displaySummary ?? config.displaySummary ?? true;
|
|
14403
|
+
let summary;
|
|
14404
|
+
if (shouldDisplaySummary) {
|
|
14405
|
+
try {
|
|
14406
|
+
summary = await ctx.experiment.summarize();
|
|
14407
|
+
} catch (error) {
|
|
14408
|
+
console.warn(
|
|
14409
|
+
"Braintrust: Failed to generate experiment summary:",
|
|
14410
|
+
error
|
|
14411
|
+
);
|
|
14412
|
+
}
|
|
14413
|
+
}
|
|
14414
|
+
await ctx.experiment.flush();
|
|
14415
|
+
if (summary && shouldDisplaySummary) {
|
|
14416
|
+
console.log(formatExperimentSummary(summary));
|
|
14417
|
+
}
|
|
14418
|
+
}
|
|
14419
|
+
};
|
|
14420
|
+
}
|
|
14421
|
+
|
|
13850
14422
|
// src/graph-framework.ts
|
|
13851
14423
|
var graph_framework_exports = {};
|
|
13852
14424
|
__export(graph_framework_exports, {
|
|
@@ -16863,5 +17435,2481 @@ var evaluatorDefinitionsSchema = import_v312.z.record(
|
|
|
16863
17435
|
evaluatorDefinitionSchema
|
|
16864
17436
|
);
|
|
16865
17437
|
|
|
17438
|
+
// src/instrumentation/core/plugin.ts
|
|
17439
|
+
var import_dc_browser = require("dc-browser");
|
|
17440
|
+
|
|
17441
|
+
// src/instrumentation/core/stream-patcher.ts
|
|
17442
|
+
function isAsyncIterable4(value) {
|
|
17443
|
+
return value !== null && typeof value === "object" && Symbol.asyncIterator in value && typeof value[Symbol.asyncIterator] === "function";
|
|
17444
|
+
}
|
|
17445
|
+
function patchStreamIfNeeded(stream, options) {
|
|
17446
|
+
if (!isAsyncIterable4(stream)) {
|
|
17447
|
+
return stream;
|
|
17448
|
+
}
|
|
17449
|
+
if (Object.isFrozen(stream) || Object.isSealed(stream)) {
|
|
17450
|
+
console.warn(
|
|
17451
|
+
"Cannot patch frozen/sealed stream. Stream output will not be collected."
|
|
17452
|
+
);
|
|
17453
|
+
return stream;
|
|
17454
|
+
}
|
|
17455
|
+
const originalIteratorFn = stream[Symbol.asyncIterator];
|
|
17456
|
+
if (originalIteratorFn.__braintrust_patched) {
|
|
17457
|
+
return stream;
|
|
17458
|
+
}
|
|
17459
|
+
try {
|
|
17460
|
+
const patchedIteratorFn = function() {
|
|
17461
|
+
const iterator = originalIteratorFn.call(this);
|
|
17462
|
+
const originalNext = iterator.next.bind(iterator);
|
|
17463
|
+
const chunks = [];
|
|
17464
|
+
let completed = false;
|
|
17465
|
+
iterator.next = async function(...args) {
|
|
17466
|
+
try {
|
|
17467
|
+
const result = await originalNext(...args);
|
|
17468
|
+
if (result.done) {
|
|
17469
|
+
if (!completed) {
|
|
17470
|
+
completed = true;
|
|
17471
|
+
try {
|
|
17472
|
+
options.onComplete(chunks);
|
|
17473
|
+
} catch (error) {
|
|
17474
|
+
console.error("Error in stream onComplete handler:", error);
|
|
17475
|
+
}
|
|
17476
|
+
}
|
|
17477
|
+
} else {
|
|
17478
|
+
const chunk = result.value;
|
|
17479
|
+
const shouldCollect = options.shouldCollect ? options.shouldCollect(chunk) : true;
|
|
17480
|
+
if (shouldCollect) {
|
|
17481
|
+
chunks.push(chunk);
|
|
17482
|
+
if (options.onChunk) {
|
|
17483
|
+
try {
|
|
17484
|
+
options.onChunk(chunk);
|
|
17485
|
+
} catch (error) {
|
|
17486
|
+
console.error("Error in stream onChunk handler:", error);
|
|
17487
|
+
}
|
|
17488
|
+
}
|
|
17489
|
+
}
|
|
17490
|
+
}
|
|
17491
|
+
return result;
|
|
17492
|
+
} catch (error) {
|
|
17493
|
+
if (!completed) {
|
|
17494
|
+
completed = true;
|
|
17495
|
+
if (options.onError) {
|
|
17496
|
+
try {
|
|
17497
|
+
options.onError(error, chunks);
|
|
17498
|
+
} catch (handlerError) {
|
|
17499
|
+
console.error("Error in stream onError handler:", handlerError);
|
|
17500
|
+
}
|
|
17501
|
+
}
|
|
17502
|
+
}
|
|
17503
|
+
throw error;
|
|
17504
|
+
}
|
|
17505
|
+
};
|
|
17506
|
+
if (iterator.return) {
|
|
17507
|
+
const originalReturn = iterator.return.bind(iterator);
|
|
17508
|
+
iterator.return = async function(...args) {
|
|
17509
|
+
if (!completed) {
|
|
17510
|
+
completed = true;
|
|
17511
|
+
try {
|
|
17512
|
+
options.onComplete(chunks);
|
|
17513
|
+
} catch (error) {
|
|
17514
|
+
console.error("Error in stream onComplete handler:", error);
|
|
17515
|
+
}
|
|
17516
|
+
}
|
|
17517
|
+
return originalReturn(...args);
|
|
17518
|
+
};
|
|
17519
|
+
}
|
|
17520
|
+
if (iterator.throw) {
|
|
17521
|
+
const originalThrow = iterator.throw.bind(iterator);
|
|
17522
|
+
iterator.throw = async function(...args) {
|
|
17523
|
+
if (!completed) {
|
|
17524
|
+
completed = true;
|
|
17525
|
+
const error = args[0];
|
|
17526
|
+
if (options.onError) {
|
|
17527
|
+
try {
|
|
17528
|
+
options.onError(error, chunks);
|
|
17529
|
+
} catch (handlerError) {
|
|
17530
|
+
console.error("Error in stream onError handler:", handlerError);
|
|
17531
|
+
}
|
|
17532
|
+
}
|
|
17533
|
+
}
|
|
17534
|
+
return originalThrow(...args);
|
|
17535
|
+
};
|
|
17536
|
+
}
|
|
17537
|
+
return iterator;
|
|
17538
|
+
};
|
|
17539
|
+
patchedIteratorFn.__braintrust_patched = true;
|
|
17540
|
+
stream[Symbol.asyncIterator] = patchedIteratorFn;
|
|
17541
|
+
return stream;
|
|
17542
|
+
} catch (error) {
|
|
17543
|
+
console.warn("Failed to patch stream:", error);
|
|
17544
|
+
return stream;
|
|
17545
|
+
}
|
|
17546
|
+
}
|
|
17547
|
+
|
|
17548
|
+
// src/instrumentation/core/plugin.ts
|
|
17549
|
+
var BasePlugin = class {
|
|
17550
|
+
enabled = false;
|
|
17551
|
+
unsubscribers = [];
|
|
17552
|
+
/**
|
|
17553
|
+
* Enables the plugin. Must be called before the plugin will receive events.
|
|
17554
|
+
*/
|
|
17555
|
+
enable() {
|
|
17556
|
+
if (this.enabled) {
|
|
17557
|
+
return;
|
|
17558
|
+
}
|
|
17559
|
+
this.enabled = true;
|
|
17560
|
+
this.onEnable();
|
|
17561
|
+
}
|
|
17562
|
+
/**
|
|
17563
|
+
* Disables the plugin. After this, the plugin will no longer receive events.
|
|
17564
|
+
*/
|
|
17565
|
+
disable() {
|
|
17566
|
+
if (!this.enabled) {
|
|
17567
|
+
return;
|
|
17568
|
+
}
|
|
17569
|
+
this.enabled = false;
|
|
17570
|
+
this.onDisable();
|
|
17571
|
+
}
|
|
17572
|
+
/**
|
|
17573
|
+
* Helper to subscribe to a channel with raw handlers.
|
|
17574
|
+
*
|
|
17575
|
+
* @param channelName - The channel name to subscribe to
|
|
17576
|
+
* @param handlers - Event handlers
|
|
17577
|
+
*/
|
|
17578
|
+
subscribe(channelName, handlers) {
|
|
17579
|
+
const channel = (0, import_dc_browser.tracingChannel)(channelName);
|
|
17580
|
+
channel.subscribe(handlers);
|
|
17581
|
+
}
|
|
17582
|
+
/**
|
|
17583
|
+
* Subscribe to a channel for async methods (non-streaming).
|
|
17584
|
+
* Creates a span and logs input/output/metrics.
|
|
17585
|
+
*/
|
|
17586
|
+
subscribeToChannel(channelName, config) {
|
|
17587
|
+
const channel = (0, import_dc_browser.tracingChannel)(channelName);
|
|
17588
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
17589
|
+
const handlers = {
|
|
17590
|
+
start: (event) => {
|
|
17591
|
+
const span = startSpan({
|
|
17592
|
+
name: config.name,
|
|
17593
|
+
spanAttributes: {
|
|
17594
|
+
type: config.type
|
|
17595
|
+
}
|
|
17596
|
+
});
|
|
17597
|
+
const startTime = getCurrentUnixTimestamp();
|
|
17598
|
+
spans.set(event, { span, startTime });
|
|
17599
|
+
try {
|
|
17600
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
17601
|
+
span.log({
|
|
17602
|
+
input,
|
|
17603
|
+
metadata
|
|
17604
|
+
});
|
|
17605
|
+
} catch (error) {
|
|
17606
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
17607
|
+
}
|
|
17608
|
+
},
|
|
17609
|
+
asyncEnd: (event) => {
|
|
17610
|
+
const spanData = spans.get(event);
|
|
17611
|
+
if (!spanData) {
|
|
17612
|
+
return;
|
|
17613
|
+
}
|
|
17614
|
+
const { span, startTime } = spanData;
|
|
17615
|
+
try {
|
|
17616
|
+
const output = config.extractOutput(event.result);
|
|
17617
|
+
const metrics = config.extractMetrics(event.result, startTime);
|
|
17618
|
+
span.log({
|
|
17619
|
+
output,
|
|
17620
|
+
metrics
|
|
17621
|
+
});
|
|
17622
|
+
} catch (error) {
|
|
17623
|
+
console.error(`Error extracting output for ${channelName}:`, error);
|
|
17624
|
+
} finally {
|
|
17625
|
+
span.end();
|
|
17626
|
+
spans.delete(event);
|
|
17627
|
+
}
|
|
17628
|
+
},
|
|
17629
|
+
error: (event) => {
|
|
17630
|
+
const spanData = spans.get(event);
|
|
17631
|
+
if (!spanData) {
|
|
17632
|
+
return;
|
|
17633
|
+
}
|
|
17634
|
+
const { span } = spanData;
|
|
17635
|
+
span.log({
|
|
17636
|
+
error: event.error.message
|
|
17637
|
+
});
|
|
17638
|
+
span.end();
|
|
17639
|
+
spans.delete(event);
|
|
17640
|
+
}
|
|
17641
|
+
};
|
|
17642
|
+
channel.subscribe(handlers);
|
|
17643
|
+
this.unsubscribers.push(() => {
|
|
17644
|
+
channel.unsubscribe(handlers);
|
|
17645
|
+
});
|
|
17646
|
+
}
|
|
17647
|
+
/**
|
|
17648
|
+
* Subscribe to a channel for async methods that may return streams.
|
|
17649
|
+
* Handles both streaming and non-streaming responses.
|
|
17650
|
+
*/
|
|
17651
|
+
subscribeToStreamingChannel(channelName, config) {
|
|
17652
|
+
const channel = (0, import_dc_browser.tracingChannel)(channelName);
|
|
17653
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
17654
|
+
const handlers = {
|
|
17655
|
+
start: (event) => {
|
|
17656
|
+
const span = startSpan({
|
|
17657
|
+
name: config.name,
|
|
17658
|
+
spanAttributes: {
|
|
17659
|
+
type: config.type
|
|
17660
|
+
}
|
|
17661
|
+
});
|
|
17662
|
+
const startTime = getCurrentUnixTimestamp();
|
|
17663
|
+
spans.set(event, { span, startTime });
|
|
17664
|
+
try {
|
|
17665
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
17666
|
+
span.log({
|
|
17667
|
+
input,
|
|
17668
|
+
metadata
|
|
17669
|
+
});
|
|
17670
|
+
} catch (error) {
|
|
17671
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
17672
|
+
}
|
|
17673
|
+
},
|
|
17674
|
+
asyncEnd: (event) => {
|
|
17675
|
+
const spanData = spans.get(event);
|
|
17676
|
+
if (!spanData) {
|
|
17677
|
+
return;
|
|
17678
|
+
}
|
|
17679
|
+
const { span, startTime } = spanData;
|
|
17680
|
+
if (isAsyncIterable4(event.result)) {
|
|
17681
|
+
patchStreamIfNeeded(event.result, {
|
|
17682
|
+
onComplete: (chunks) => {
|
|
17683
|
+
try {
|
|
17684
|
+
let output;
|
|
17685
|
+
let metrics;
|
|
17686
|
+
if (config.aggregateChunks) {
|
|
17687
|
+
const aggregated = config.aggregateChunks(chunks);
|
|
17688
|
+
output = aggregated.output;
|
|
17689
|
+
metrics = aggregated.metrics;
|
|
17690
|
+
} else {
|
|
17691
|
+
output = config.extractOutput(chunks);
|
|
17692
|
+
metrics = config.extractMetrics(chunks, startTime);
|
|
17693
|
+
}
|
|
17694
|
+
if (!metrics.time_to_first_token && chunks.length > 0) {
|
|
17695
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
17696
|
+
}
|
|
17697
|
+
span.log({
|
|
17698
|
+
output,
|
|
17699
|
+
metrics
|
|
17700
|
+
});
|
|
17701
|
+
} catch (error) {
|
|
17702
|
+
console.error(
|
|
17703
|
+
`Error extracting output for ${channelName}:`,
|
|
17704
|
+
error
|
|
17705
|
+
);
|
|
17706
|
+
} finally {
|
|
17707
|
+
span.end();
|
|
17708
|
+
}
|
|
17709
|
+
},
|
|
17710
|
+
onError: (error) => {
|
|
17711
|
+
span.log({
|
|
17712
|
+
error: error.message
|
|
17713
|
+
});
|
|
17714
|
+
span.end();
|
|
17715
|
+
}
|
|
17716
|
+
});
|
|
17717
|
+
} else {
|
|
17718
|
+
try {
|
|
17719
|
+
const output = config.extractOutput(event.result);
|
|
17720
|
+
const metrics = config.extractMetrics(event.result, startTime);
|
|
17721
|
+
span.log({
|
|
17722
|
+
output,
|
|
17723
|
+
metrics
|
|
17724
|
+
});
|
|
17725
|
+
} catch (error) {
|
|
17726
|
+
console.error(`Error extracting output for ${channelName}:`, error);
|
|
17727
|
+
} finally {
|
|
17728
|
+
span.end();
|
|
17729
|
+
spans.delete(event);
|
|
17730
|
+
}
|
|
17731
|
+
}
|
|
17732
|
+
},
|
|
17733
|
+
error: (event) => {
|
|
17734
|
+
const spanData = spans.get(event);
|
|
17735
|
+
if (!spanData) {
|
|
17736
|
+
return;
|
|
17737
|
+
}
|
|
17738
|
+
const { span } = spanData;
|
|
17739
|
+
span.log({
|
|
17740
|
+
error: event.error.message
|
|
17741
|
+
});
|
|
17742
|
+
span.end();
|
|
17743
|
+
spans.delete(event);
|
|
17744
|
+
}
|
|
17745
|
+
};
|
|
17746
|
+
channel.subscribe(handlers);
|
|
17747
|
+
this.unsubscribers.push(() => {
|
|
17748
|
+
channel.unsubscribe(handlers);
|
|
17749
|
+
});
|
|
17750
|
+
}
|
|
17751
|
+
/**
|
|
17752
|
+
* Subscribe to a channel for sync methods that return event-based streams.
|
|
17753
|
+
* Used for methods like beta.chat.completions.stream() and responses.stream().
|
|
17754
|
+
*/
|
|
17755
|
+
subscribeToSyncStreamChannel(channelName, config) {
|
|
17756
|
+
const channel = (0, import_dc_browser.tracingChannel)(channelName);
|
|
17757
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
17758
|
+
const handlers = {
|
|
17759
|
+
start: (event) => {
|
|
17760
|
+
const span = startSpan({
|
|
17761
|
+
name: config.name,
|
|
17762
|
+
spanAttributes: {
|
|
17763
|
+
type: config.type
|
|
17764
|
+
}
|
|
17765
|
+
});
|
|
17766
|
+
const startTime = getCurrentUnixTimestamp();
|
|
17767
|
+
spans.set(event, { span, startTime });
|
|
17768
|
+
try {
|
|
17769
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
17770
|
+
span.log({
|
|
17771
|
+
input,
|
|
17772
|
+
metadata
|
|
17773
|
+
});
|
|
17774
|
+
} catch (error) {
|
|
17775
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
17776
|
+
}
|
|
17777
|
+
},
|
|
17778
|
+
end: (event) => {
|
|
17779
|
+
const spanData = spans.get(event);
|
|
17780
|
+
if (!spanData) {
|
|
17781
|
+
return;
|
|
17782
|
+
}
|
|
17783
|
+
const { span, startTime } = spanData;
|
|
17784
|
+
const stream = event.result;
|
|
17785
|
+
if (!stream || typeof stream.on !== "function") {
|
|
17786
|
+
span.end();
|
|
17787
|
+
spans.delete(event);
|
|
17788
|
+
return;
|
|
17789
|
+
}
|
|
17790
|
+
let first = true;
|
|
17791
|
+
stream.on("chunk", (chunk) => {
|
|
17792
|
+
if (first) {
|
|
17793
|
+
const now2 = getCurrentUnixTimestamp();
|
|
17794
|
+
span.log({
|
|
17795
|
+
metrics: {
|
|
17796
|
+
time_to_first_token: now2 - startTime
|
|
17797
|
+
}
|
|
17798
|
+
});
|
|
17799
|
+
first = false;
|
|
17800
|
+
}
|
|
17801
|
+
});
|
|
17802
|
+
stream.on("chatCompletion", (completion) => {
|
|
17803
|
+
try {
|
|
17804
|
+
span.log({
|
|
17805
|
+
output: completion.choices
|
|
17806
|
+
});
|
|
17807
|
+
} catch (error) {
|
|
17808
|
+
console.error(
|
|
17809
|
+
`Error extracting chatCompletion for ${channelName}:`,
|
|
17810
|
+
error
|
|
17811
|
+
);
|
|
17812
|
+
}
|
|
17813
|
+
});
|
|
17814
|
+
stream.on("event", (streamEvent) => {
|
|
17815
|
+
if (config.extractFromEvent) {
|
|
17816
|
+
try {
|
|
17817
|
+
if (first) {
|
|
17818
|
+
const now2 = getCurrentUnixTimestamp();
|
|
17819
|
+
span.log({
|
|
17820
|
+
metrics: {
|
|
17821
|
+
time_to_first_token: now2 - startTime
|
|
17822
|
+
}
|
|
17823
|
+
});
|
|
17824
|
+
first = false;
|
|
17825
|
+
}
|
|
17826
|
+
const extracted = config.extractFromEvent(streamEvent);
|
|
17827
|
+
if (extracted && Object.keys(extracted).length > 0) {
|
|
17828
|
+
span.log(extracted);
|
|
17829
|
+
}
|
|
17830
|
+
} catch (error) {
|
|
17831
|
+
console.error(
|
|
17832
|
+
`Error extracting event for ${channelName}:`,
|
|
17833
|
+
error
|
|
17834
|
+
);
|
|
17835
|
+
}
|
|
17836
|
+
}
|
|
17837
|
+
});
|
|
17838
|
+
stream.on("end", () => {
|
|
17839
|
+
span.end();
|
|
17840
|
+
spans.delete(event);
|
|
17841
|
+
});
|
|
17842
|
+
},
|
|
17843
|
+
error: (event) => {
|
|
17844
|
+
const spanData = spans.get(event);
|
|
17845
|
+
if (!spanData) {
|
|
17846
|
+
return;
|
|
17847
|
+
}
|
|
17848
|
+
const { span } = spanData;
|
|
17849
|
+
span.log({
|
|
17850
|
+
error: event.error.message
|
|
17851
|
+
});
|
|
17852
|
+
span.end();
|
|
17853
|
+
spans.delete(event);
|
|
17854
|
+
}
|
|
17855
|
+
};
|
|
17856
|
+
channel.subscribe(handlers);
|
|
17857
|
+
this.unsubscribers.push(() => {
|
|
17858
|
+
channel.unsubscribe(handlers);
|
|
17859
|
+
});
|
|
17860
|
+
}
|
|
17861
|
+
};
|
|
17862
|
+
|
|
17863
|
+
// src/instrumentation/plugins/openai-plugin.ts
|
|
17864
|
+
var OpenAIPlugin = class extends BasePlugin {
|
|
17865
|
+
constructor() {
|
|
17866
|
+
super();
|
|
17867
|
+
}
|
|
17868
|
+
onEnable() {
|
|
17869
|
+
this.subscribeToStreamingChannel(
|
|
17870
|
+
"orchestrion:openai:chat.completions.create",
|
|
17871
|
+
{
|
|
17872
|
+
name: "Chat Completion",
|
|
17873
|
+
type: "llm" /* LLM */,
|
|
17874
|
+
extractInput: (args) => {
|
|
17875
|
+
const params = args[0] || {};
|
|
17876
|
+
const { messages, ...metadata } = params;
|
|
17877
|
+
return {
|
|
17878
|
+
input: processInputAttachments(messages),
|
|
17879
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17880
|
+
};
|
|
17881
|
+
},
|
|
17882
|
+
extractOutput: (result) => {
|
|
17883
|
+
return result?.choices;
|
|
17884
|
+
},
|
|
17885
|
+
extractMetrics: (result, startTime) => {
|
|
17886
|
+
const metrics = parseMetricsFromUsage3(result?.usage);
|
|
17887
|
+
if (startTime) {
|
|
17888
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
17889
|
+
}
|
|
17890
|
+
return metrics;
|
|
17891
|
+
},
|
|
17892
|
+
aggregateChunks: aggregateChatCompletionChunks
|
|
17893
|
+
}
|
|
17894
|
+
);
|
|
17895
|
+
this.subscribeToChannel("orchestrion:openai:embeddings.create", {
|
|
17896
|
+
name: "Embedding",
|
|
17897
|
+
type: "llm" /* LLM */,
|
|
17898
|
+
extractInput: (args) => {
|
|
17899
|
+
const params = args[0] || {};
|
|
17900
|
+
const { input, ...metadata } = params;
|
|
17901
|
+
return {
|
|
17902
|
+
input,
|
|
17903
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17904
|
+
};
|
|
17905
|
+
},
|
|
17906
|
+
extractOutput: (result) => {
|
|
17907
|
+
return result?.data?.map((d) => d.embedding);
|
|
17908
|
+
},
|
|
17909
|
+
extractMetrics: (result) => {
|
|
17910
|
+
return parseMetricsFromUsage3(result?.usage);
|
|
17911
|
+
}
|
|
17912
|
+
});
|
|
17913
|
+
this.subscribeToStreamingChannel(
|
|
17914
|
+
"orchestrion:openai:beta.chat.completions.parse",
|
|
17915
|
+
{
|
|
17916
|
+
name: "Chat Completion",
|
|
17917
|
+
type: "llm" /* LLM */,
|
|
17918
|
+
extractInput: (args) => {
|
|
17919
|
+
const params = args[0] || {};
|
|
17920
|
+
const { messages, ...metadata } = params;
|
|
17921
|
+
return {
|
|
17922
|
+
input: processInputAttachments(messages),
|
|
17923
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17924
|
+
};
|
|
17925
|
+
},
|
|
17926
|
+
extractOutput: (result) => {
|
|
17927
|
+
return result?.choices;
|
|
17928
|
+
},
|
|
17929
|
+
extractMetrics: (result, startTime) => {
|
|
17930
|
+
const metrics = parseMetricsFromUsage3(result?.usage);
|
|
17931
|
+
if (startTime) {
|
|
17932
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
17933
|
+
}
|
|
17934
|
+
return metrics;
|
|
17935
|
+
},
|
|
17936
|
+
aggregateChunks: aggregateChatCompletionChunks
|
|
17937
|
+
}
|
|
17938
|
+
);
|
|
17939
|
+
this.subscribeToSyncStreamChannel(
|
|
17940
|
+
"orchestrion:openai:beta.chat.completions.stream",
|
|
17941
|
+
{
|
|
17942
|
+
name: "Chat Completion",
|
|
17943
|
+
type: "llm" /* LLM */,
|
|
17944
|
+
extractInput: (args) => {
|
|
17945
|
+
const params = args[0] || {};
|
|
17946
|
+
const { messages, ...metadata } = params;
|
|
17947
|
+
return {
|
|
17948
|
+
input: processInputAttachments(messages),
|
|
17949
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17950
|
+
};
|
|
17951
|
+
}
|
|
17952
|
+
}
|
|
17953
|
+
);
|
|
17954
|
+
this.subscribeToChannel("orchestrion:openai:moderations.create", {
|
|
17955
|
+
name: "Moderation",
|
|
17956
|
+
type: "llm" /* LLM */,
|
|
17957
|
+
extractInput: (args) => {
|
|
17958
|
+
const params = args[0] || {};
|
|
17959
|
+
const { input, ...metadata } = params;
|
|
17960
|
+
return {
|
|
17961
|
+
input,
|
|
17962
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17963
|
+
};
|
|
17964
|
+
},
|
|
17965
|
+
extractOutput: (result) => {
|
|
17966
|
+
return result?.results;
|
|
17967
|
+
},
|
|
17968
|
+
extractMetrics: () => {
|
|
17969
|
+
return {};
|
|
17970
|
+
}
|
|
17971
|
+
});
|
|
17972
|
+
this.subscribeToStreamingChannel("orchestrion:openai:responses.create", {
|
|
17973
|
+
name: "openai.responses.create",
|
|
17974
|
+
type: "llm" /* LLM */,
|
|
17975
|
+
extractInput: (args) => {
|
|
17976
|
+
const params = args[0] || {};
|
|
17977
|
+
const { input, ...metadata } = params;
|
|
17978
|
+
return {
|
|
17979
|
+
input: processInputAttachments(input),
|
|
17980
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17981
|
+
};
|
|
17982
|
+
},
|
|
17983
|
+
extractOutput: (result) => {
|
|
17984
|
+
return processImagesInOutput2(result?.output);
|
|
17985
|
+
},
|
|
17986
|
+
extractMetrics: (result, startTime) => {
|
|
17987
|
+
const metrics = parseMetricsFromUsage3(result?.usage);
|
|
17988
|
+
if (startTime) {
|
|
17989
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
17990
|
+
}
|
|
17991
|
+
return metrics;
|
|
17992
|
+
}
|
|
17993
|
+
});
|
|
17994
|
+
this.subscribeToSyncStreamChannel("orchestrion:openai:responses.stream", {
|
|
17995
|
+
name: "openai.responses.stream",
|
|
17996
|
+
type: "llm" /* LLM */,
|
|
17997
|
+
extractInput: (args) => {
|
|
17998
|
+
const params = args[0] || {};
|
|
17999
|
+
const { input, ...metadata } = params;
|
|
18000
|
+
return {
|
|
18001
|
+
input: processInputAttachments(input),
|
|
18002
|
+
metadata: { ...metadata, provider: "openai" }
|
|
18003
|
+
};
|
|
18004
|
+
},
|
|
18005
|
+
extractFromEvent: (event) => {
|
|
18006
|
+
if (!event || !event.type || !event.response) {
|
|
18007
|
+
return {};
|
|
18008
|
+
}
|
|
18009
|
+
const response = event.response;
|
|
18010
|
+
if (event.type === "response.completed") {
|
|
18011
|
+
const data = {};
|
|
18012
|
+
if (response?.output !== void 0) {
|
|
18013
|
+
data.output = processImagesInOutput2(response.output);
|
|
18014
|
+
}
|
|
18015
|
+
if (response) {
|
|
18016
|
+
const { usage: _usage, output: _output, ...metadata } = response;
|
|
18017
|
+
if (Object.keys(metadata).length > 0) {
|
|
18018
|
+
data.metadata = metadata;
|
|
18019
|
+
}
|
|
18020
|
+
}
|
|
18021
|
+
data.metrics = parseMetricsFromUsage3(response?.usage);
|
|
18022
|
+
return data;
|
|
18023
|
+
}
|
|
18024
|
+
return {};
|
|
18025
|
+
}
|
|
18026
|
+
});
|
|
18027
|
+
this.subscribeToStreamingChannel("orchestrion:openai:responses.parse", {
|
|
18028
|
+
name: "openai.responses.parse",
|
|
18029
|
+
type: "llm" /* LLM */,
|
|
18030
|
+
extractInput: (args) => {
|
|
18031
|
+
const params = args[0] || {};
|
|
18032
|
+
const { input, ...metadata } = params;
|
|
18033
|
+
return {
|
|
18034
|
+
input: processInputAttachments(input),
|
|
18035
|
+
metadata: { ...metadata, provider: "openai" }
|
|
18036
|
+
};
|
|
18037
|
+
},
|
|
18038
|
+
extractOutput: (result) => {
|
|
18039
|
+
return processImagesInOutput2(result?.output);
|
|
18040
|
+
},
|
|
18041
|
+
extractMetrics: (result, startTime) => {
|
|
18042
|
+
const metrics = parseMetricsFromUsage3(result?.usage);
|
|
18043
|
+
if (startTime) {
|
|
18044
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18045
|
+
}
|
|
18046
|
+
return metrics;
|
|
18047
|
+
}
|
|
18048
|
+
});
|
|
18049
|
+
}
|
|
18050
|
+
onDisable() {
|
|
18051
|
+
}
|
|
18052
|
+
};
|
|
18053
|
+
var TOKEN_NAME_MAP2 = {
|
|
18054
|
+
input_tokens: "prompt_tokens",
|
|
18055
|
+
output_tokens: "completion_tokens",
|
|
18056
|
+
total_tokens: "tokens"
|
|
18057
|
+
};
|
|
18058
|
+
var TOKEN_PREFIX_MAP2 = {
|
|
18059
|
+
input: "prompt",
|
|
18060
|
+
output: "completion"
|
|
18061
|
+
};
|
|
18062
|
+
function parseMetricsFromUsage3(usage) {
|
|
18063
|
+
if (!usage) {
|
|
18064
|
+
return {};
|
|
18065
|
+
}
|
|
18066
|
+
const metrics = {};
|
|
18067
|
+
for (const [oai_name, value] of Object.entries(usage)) {
|
|
18068
|
+
if (typeof value === "number") {
|
|
18069
|
+
const metricName = TOKEN_NAME_MAP2[oai_name] || oai_name;
|
|
18070
|
+
metrics[metricName] = value;
|
|
18071
|
+
} else if (oai_name.endsWith("_tokens_details")) {
|
|
18072
|
+
if (!isObject(value)) {
|
|
18073
|
+
continue;
|
|
18074
|
+
}
|
|
18075
|
+
const rawPrefix = oai_name.slice(0, -"_tokens_details".length);
|
|
18076
|
+
const prefix = TOKEN_PREFIX_MAP2[rawPrefix] || rawPrefix;
|
|
18077
|
+
for (const [key, n] of Object.entries(value)) {
|
|
18078
|
+
if (typeof n !== "number") {
|
|
18079
|
+
continue;
|
|
18080
|
+
}
|
|
18081
|
+
const metricName = `${prefix}_${key}`;
|
|
18082
|
+
metrics[metricName] = n;
|
|
18083
|
+
}
|
|
18084
|
+
}
|
|
18085
|
+
}
|
|
18086
|
+
return metrics;
|
|
18087
|
+
}
|
|
18088
|
+
function processImagesInOutput2(output) {
|
|
18089
|
+
if (Array.isArray(output)) {
|
|
18090
|
+
return output.map(processImagesInOutput2);
|
|
18091
|
+
}
|
|
18092
|
+
if (isObject(output)) {
|
|
18093
|
+
if (output.type === "image_generation_call" && output.result && typeof output.result === "string") {
|
|
18094
|
+
const fileExtension = output.output_format || "png";
|
|
18095
|
+
const contentType = `image/${fileExtension}`;
|
|
18096
|
+
const baseFilename = output.revised_prompt && typeof output.revised_prompt === "string" ? output.revised_prompt.slice(0, 50).replace(/[^a-zA-Z0-9]/g, "_") : "generated_image";
|
|
18097
|
+
const filename = `${baseFilename}.${fileExtension}`;
|
|
18098
|
+
const binaryString = atob(output.result);
|
|
18099
|
+
const bytes = new Uint8Array(binaryString.length);
|
|
18100
|
+
for (let i = 0; i < binaryString.length; i++) {
|
|
18101
|
+
bytes[i] = binaryString.charCodeAt(i);
|
|
18102
|
+
}
|
|
18103
|
+
const blob = new Blob([bytes], { type: contentType });
|
|
18104
|
+
const attachment = new Attachment({
|
|
18105
|
+
data: blob,
|
|
18106
|
+
filename,
|
|
18107
|
+
contentType
|
|
18108
|
+
});
|
|
18109
|
+
return {
|
|
18110
|
+
...output,
|
|
18111
|
+
result: attachment
|
|
18112
|
+
};
|
|
18113
|
+
}
|
|
18114
|
+
}
|
|
18115
|
+
return output;
|
|
18116
|
+
}
|
|
18117
|
+
function aggregateChatCompletionChunks(chunks) {
|
|
18118
|
+
let role = void 0;
|
|
18119
|
+
let content = void 0;
|
|
18120
|
+
let tool_calls = void 0;
|
|
18121
|
+
let finish_reason = void 0;
|
|
18122
|
+
let metrics = {};
|
|
18123
|
+
for (const chunk of chunks) {
|
|
18124
|
+
if (chunk.usage) {
|
|
18125
|
+
metrics = {
|
|
18126
|
+
...metrics,
|
|
18127
|
+
...parseMetricsFromUsage3(chunk.usage)
|
|
18128
|
+
};
|
|
18129
|
+
}
|
|
18130
|
+
const delta = chunk.choices?.[0]?.delta;
|
|
18131
|
+
if (!delta) {
|
|
18132
|
+
continue;
|
|
18133
|
+
}
|
|
18134
|
+
if (!role && delta.role) {
|
|
18135
|
+
role = delta.role;
|
|
18136
|
+
}
|
|
18137
|
+
if (delta.finish_reason) {
|
|
18138
|
+
finish_reason = delta.finish_reason;
|
|
18139
|
+
}
|
|
18140
|
+
if (delta.content) {
|
|
18141
|
+
content = (content || "") + delta.content;
|
|
18142
|
+
}
|
|
18143
|
+
if (delta.tool_calls) {
|
|
18144
|
+
const toolDelta = delta.tool_calls[0];
|
|
18145
|
+
if (!tool_calls || toolDelta.id && tool_calls[tool_calls.length - 1].id !== toolDelta.id) {
|
|
18146
|
+
tool_calls = [
|
|
18147
|
+
...tool_calls || [],
|
|
18148
|
+
{
|
|
18149
|
+
id: toolDelta.id,
|
|
18150
|
+
type: toolDelta.type,
|
|
18151
|
+
function: toolDelta.function
|
|
18152
|
+
}
|
|
18153
|
+
];
|
|
18154
|
+
} else {
|
|
18155
|
+
tool_calls[tool_calls.length - 1].function.arguments += toolDelta.function.arguments;
|
|
18156
|
+
}
|
|
18157
|
+
}
|
|
18158
|
+
}
|
|
18159
|
+
return {
|
|
18160
|
+
metrics,
|
|
18161
|
+
output: [
|
|
18162
|
+
{
|
|
18163
|
+
index: 0,
|
|
18164
|
+
message: {
|
|
18165
|
+
role,
|
|
18166
|
+
content,
|
|
18167
|
+
tool_calls
|
|
18168
|
+
},
|
|
18169
|
+
logprobs: null,
|
|
18170
|
+
finish_reason
|
|
18171
|
+
}
|
|
18172
|
+
]
|
|
18173
|
+
};
|
|
18174
|
+
}
|
|
18175
|
+
|
|
18176
|
+
// src/instrumentation/plugins/anthropic-plugin.ts
|
|
18177
|
+
var import_dc_browser2 = require("dc-browser");
|
|
18178
|
+
var AnthropicPlugin = class extends BasePlugin {
|
|
18179
|
+
unsubscribers = [];
|
|
18180
|
+
onEnable() {
|
|
18181
|
+
this.subscribeToAnthropicChannels();
|
|
18182
|
+
}
|
|
18183
|
+
onDisable() {
|
|
18184
|
+
for (const unsubscribe of this.unsubscribers) {
|
|
18185
|
+
unsubscribe();
|
|
18186
|
+
}
|
|
18187
|
+
this.unsubscribers = [];
|
|
18188
|
+
}
|
|
18189
|
+
subscribeToAnthropicChannels() {
|
|
18190
|
+
this.subscribeToStreamingChannel("orchestrion:anthropic:messages.create", {
|
|
18191
|
+
name: "anthropic.messages.create",
|
|
18192
|
+
type: "llm" /* LLM */,
|
|
18193
|
+
extractInput: (args) => {
|
|
18194
|
+
const params = args[0] || {};
|
|
18195
|
+
const input = coalesceInput2(params.messages || [], params.system);
|
|
18196
|
+
const metadata = filterFrom2(params, ["messages", "system"]);
|
|
18197
|
+
return {
|
|
18198
|
+
input: processAttachmentsInInput3(input),
|
|
18199
|
+
metadata: { ...metadata, provider: "anthropic" }
|
|
18200
|
+
};
|
|
18201
|
+
},
|
|
18202
|
+
extractOutput: (result) => {
|
|
18203
|
+
return result ? { role: result.role, content: result.content } : null;
|
|
18204
|
+
},
|
|
18205
|
+
extractMetrics: (result, startTime) => {
|
|
18206
|
+
const metrics = parseMetricsFromUsage4(result?.usage);
|
|
18207
|
+
if (startTime) {
|
|
18208
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18209
|
+
}
|
|
18210
|
+
const finalized = finalizeAnthropicTokens(metrics);
|
|
18211
|
+
return Object.fromEntries(
|
|
18212
|
+
Object.entries(finalized).filter(([, v]) => v !== void 0)
|
|
18213
|
+
);
|
|
18214
|
+
},
|
|
18215
|
+
extractMetadata: (result) => {
|
|
18216
|
+
const metadata = {};
|
|
18217
|
+
const metas = ["stop_reason", "stop_sequence"];
|
|
18218
|
+
for (const m of metas) {
|
|
18219
|
+
if (result?.[m] !== void 0) {
|
|
18220
|
+
metadata[m] = result[m];
|
|
18221
|
+
}
|
|
18222
|
+
}
|
|
18223
|
+
return metadata;
|
|
18224
|
+
},
|
|
18225
|
+
aggregateChunks: aggregateAnthropicStreamChunks,
|
|
18226
|
+
isStreaming: (args) => {
|
|
18227
|
+
return args[0]?.stream === true;
|
|
18228
|
+
}
|
|
18229
|
+
});
|
|
18230
|
+
this.subscribeToStreamingChannel(
|
|
18231
|
+
"orchestrion:anthropic:beta.messages.create",
|
|
18232
|
+
{
|
|
18233
|
+
name: "anthropic.beta.messages.create",
|
|
18234
|
+
type: "llm" /* LLM */,
|
|
18235
|
+
extractInput: (args) => {
|
|
18236
|
+
const params = args[0] || {};
|
|
18237
|
+
const input = coalesceInput2(params.messages || [], params.system);
|
|
18238
|
+
const metadata = filterFrom2(params, ["messages", "system"]);
|
|
18239
|
+
return {
|
|
18240
|
+
input: processAttachmentsInInput3(input),
|
|
18241
|
+
metadata: { ...metadata, provider: "anthropic" }
|
|
18242
|
+
};
|
|
18243
|
+
},
|
|
18244
|
+
extractOutput: (result) => {
|
|
18245
|
+
return result ? { role: result.role, content: result.content } : null;
|
|
18246
|
+
},
|
|
18247
|
+
extractMetrics: (result, startTime) => {
|
|
18248
|
+
const metrics = parseMetricsFromUsage4(result?.usage);
|
|
18249
|
+
if (startTime) {
|
|
18250
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18251
|
+
}
|
|
18252
|
+
const finalized = finalizeAnthropicTokens(metrics);
|
|
18253
|
+
return Object.fromEntries(
|
|
18254
|
+
Object.entries(finalized).filter(([, v]) => v !== void 0)
|
|
18255
|
+
);
|
|
18256
|
+
},
|
|
18257
|
+
extractMetadata: (result) => {
|
|
18258
|
+
const metadata = {};
|
|
18259
|
+
const metas = ["stop_reason", "stop_sequence"];
|
|
18260
|
+
for (const m of metas) {
|
|
18261
|
+
if (result?.[m] !== void 0) {
|
|
18262
|
+
metadata[m] = result[m];
|
|
18263
|
+
}
|
|
18264
|
+
}
|
|
18265
|
+
return metadata;
|
|
18266
|
+
},
|
|
18267
|
+
aggregateChunks: aggregateAnthropicStreamChunks,
|
|
18268
|
+
isStreaming: (args) => {
|
|
18269
|
+
return args[0]?.stream === true;
|
|
18270
|
+
}
|
|
18271
|
+
}
|
|
18272
|
+
);
|
|
18273
|
+
}
|
|
18274
|
+
/**
|
|
18275
|
+
* Subscribe to a channel for async methods that may return streams.
|
|
18276
|
+
* Handles both streaming and non-streaming responses based on the stream parameter.
|
|
18277
|
+
*/
|
|
18278
|
+
subscribeToStreamingChannel(channelName, config) {
|
|
18279
|
+
const channel = (0, import_dc_browser2.tracingChannel)(channelName);
|
|
18280
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
18281
|
+
const handlers = {
|
|
18282
|
+
start: (event) => {
|
|
18283
|
+
const span = startSpan({
|
|
18284
|
+
name: config.name,
|
|
18285
|
+
spanAttributes: {
|
|
18286
|
+
type: config.type
|
|
18287
|
+
}
|
|
18288
|
+
});
|
|
18289
|
+
const startTime = getCurrentUnixTimestamp();
|
|
18290
|
+
spans.set(event, { span, startTime });
|
|
18291
|
+
try {
|
|
18292
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
18293
|
+
span.log({
|
|
18294
|
+
input,
|
|
18295
|
+
metadata
|
|
18296
|
+
});
|
|
18297
|
+
} catch (error) {
|
|
18298
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
18299
|
+
}
|
|
18300
|
+
},
|
|
18301
|
+
asyncEnd: (event) => {
|
|
18302
|
+
const spanData = spans.get(event);
|
|
18303
|
+
if (!spanData) {
|
|
18304
|
+
return;
|
|
18305
|
+
}
|
|
18306
|
+
const { span, startTime } = spanData;
|
|
18307
|
+
const isStreaming = config.isStreaming ? config.isStreaming(event.arguments) : isAsyncIterable4(event.result);
|
|
18308
|
+
if (isStreaming && isAsyncIterable4(event.result)) {
|
|
18309
|
+
patchStreamIfNeeded(event.result, {
|
|
18310
|
+
onComplete: (chunks) => {
|
|
18311
|
+
try {
|
|
18312
|
+
let output;
|
|
18313
|
+
let metrics;
|
|
18314
|
+
let metadata = {};
|
|
18315
|
+
if (config.aggregateChunks) {
|
|
18316
|
+
const aggregated = config.aggregateChunks(chunks);
|
|
18317
|
+
output = aggregated.output;
|
|
18318
|
+
metrics = aggregated.metrics;
|
|
18319
|
+
metadata = aggregated.metadata || {};
|
|
18320
|
+
} else {
|
|
18321
|
+
output = config.extractOutput(chunks);
|
|
18322
|
+
metrics = config.extractMetrics(chunks, startTime);
|
|
18323
|
+
if (config.extractMetadata) {
|
|
18324
|
+
metadata = config.extractMetadata(chunks);
|
|
18325
|
+
}
|
|
18326
|
+
}
|
|
18327
|
+
if (!metrics.time_to_first_token && chunks.length > 0) {
|
|
18328
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18329
|
+
}
|
|
18330
|
+
span.log({
|
|
18331
|
+
output,
|
|
18332
|
+
metrics,
|
|
18333
|
+
metadata
|
|
18334
|
+
});
|
|
18335
|
+
} catch (error) {
|
|
18336
|
+
console.error(
|
|
18337
|
+
`Error extracting output for ${channelName}:`,
|
|
18338
|
+
error
|
|
18339
|
+
);
|
|
18340
|
+
} finally {
|
|
18341
|
+
span.end();
|
|
18342
|
+
}
|
|
18343
|
+
},
|
|
18344
|
+
onError: (error) => {
|
|
18345
|
+
span.log({
|
|
18346
|
+
error: error.message
|
|
18347
|
+
});
|
|
18348
|
+
span.end();
|
|
18349
|
+
}
|
|
18350
|
+
});
|
|
18351
|
+
} else {
|
|
18352
|
+
try {
|
|
18353
|
+
const output = config.extractOutput(event.result);
|
|
18354
|
+
const metrics = config.extractMetrics(event.result, startTime);
|
|
18355
|
+
const metadata = config.extractMetadata ? config.extractMetadata(event.result) : {};
|
|
18356
|
+
span.log({
|
|
18357
|
+
output,
|
|
18358
|
+
metrics,
|
|
18359
|
+
metadata
|
|
18360
|
+
});
|
|
18361
|
+
} catch (error) {
|
|
18362
|
+
console.error(`Error extracting output for ${channelName}:`, error);
|
|
18363
|
+
} finally {
|
|
18364
|
+
span.end();
|
|
18365
|
+
spans.delete(event);
|
|
18366
|
+
}
|
|
18367
|
+
}
|
|
18368
|
+
},
|
|
18369
|
+
error: (event) => {
|
|
18370
|
+
const spanData = spans.get(event);
|
|
18371
|
+
if (!spanData) {
|
|
18372
|
+
return;
|
|
18373
|
+
}
|
|
18374
|
+
const { span } = spanData;
|
|
18375
|
+
span.log({
|
|
18376
|
+
error: event.error.message
|
|
18377
|
+
});
|
|
18378
|
+
span.end();
|
|
18379
|
+
spans.delete(event);
|
|
18380
|
+
}
|
|
18381
|
+
};
|
|
18382
|
+
channel.subscribe(handlers);
|
|
18383
|
+
this.unsubscribers.push(() => {
|
|
18384
|
+
channel.unsubscribe(handlers);
|
|
18385
|
+
});
|
|
18386
|
+
}
|
|
18387
|
+
};
|
|
18388
|
+
function parseMetricsFromUsage4(usage) {
|
|
18389
|
+
if (!usage) {
|
|
18390
|
+
return {};
|
|
18391
|
+
}
|
|
18392
|
+
const metrics = {};
|
|
18393
|
+
function saveIfExistsTo(source, target) {
|
|
18394
|
+
const value = usage[source];
|
|
18395
|
+
if (value !== void 0 && value !== null && typeof value === "number") {
|
|
18396
|
+
metrics[target] = value;
|
|
18397
|
+
}
|
|
18398
|
+
}
|
|
18399
|
+
saveIfExistsTo("input_tokens", "prompt_tokens");
|
|
18400
|
+
saveIfExistsTo("output_tokens", "completion_tokens");
|
|
18401
|
+
saveIfExistsTo("cache_read_input_tokens", "prompt_cached_tokens");
|
|
18402
|
+
saveIfExistsTo("cache_creation_input_tokens", "prompt_cache_creation_tokens");
|
|
18403
|
+
return metrics;
|
|
18404
|
+
}
|
|
18405
|
+
function aggregateAnthropicStreamChunks(chunks) {
|
|
18406
|
+
const deltas = [];
|
|
18407
|
+
let metrics = {};
|
|
18408
|
+
let metadata = {};
|
|
18409
|
+
for (const chunk of chunks) {
|
|
18410
|
+
switch (chunk?.type) {
|
|
18411
|
+
case "message_start":
|
|
18412
|
+
if (chunk.message?.usage) {
|
|
18413
|
+
const initialMetrics = parseMetricsFromUsage4(chunk.message.usage);
|
|
18414
|
+
metrics = { ...metrics, ...initialMetrics };
|
|
18415
|
+
}
|
|
18416
|
+
break;
|
|
18417
|
+
case "content_block_delta":
|
|
18418
|
+
if (chunk.delta?.type === "text_delta") {
|
|
18419
|
+
const text = chunk.delta?.text;
|
|
18420
|
+
if (text) {
|
|
18421
|
+
deltas.push(text);
|
|
18422
|
+
}
|
|
18423
|
+
}
|
|
18424
|
+
break;
|
|
18425
|
+
case "message_delta":
|
|
18426
|
+
if (chunk.usage) {
|
|
18427
|
+
const finalMetrics = parseMetricsFromUsage4(chunk.usage);
|
|
18428
|
+
metrics = { ...metrics, ...finalMetrics };
|
|
18429
|
+
}
|
|
18430
|
+
if (chunk.delta) {
|
|
18431
|
+
metadata = { ...metadata, ...chunk.delta };
|
|
18432
|
+
}
|
|
18433
|
+
break;
|
|
18434
|
+
}
|
|
18435
|
+
}
|
|
18436
|
+
const output = deltas.join("");
|
|
18437
|
+
const finalized = finalizeAnthropicTokens(metrics);
|
|
18438
|
+
const filteredMetrics = Object.fromEntries(
|
|
18439
|
+
Object.entries(finalized).filter(([, v]) => v !== void 0)
|
|
18440
|
+
);
|
|
18441
|
+
return {
|
|
18442
|
+
output,
|
|
18443
|
+
metrics: filteredMetrics,
|
|
18444
|
+
metadata
|
|
18445
|
+
};
|
|
18446
|
+
}
|
|
18447
|
+
function convertBase64ToAttachment2(source, contentType) {
|
|
18448
|
+
const mediaType = typeof source.media_type === "string" ? source.media_type : "image/png";
|
|
18449
|
+
const base64Data = source.data;
|
|
18450
|
+
if (base64Data && typeof base64Data === "string") {
|
|
18451
|
+
const binaryString = atob(base64Data);
|
|
18452
|
+
const bytes = new Uint8Array(binaryString.length);
|
|
18453
|
+
for (let i = 0; i < binaryString.length; i++) {
|
|
18454
|
+
bytes[i] = binaryString.charCodeAt(i);
|
|
18455
|
+
}
|
|
18456
|
+
const blob = new Blob([bytes], { type: mediaType });
|
|
18457
|
+
const extension = mediaType.split("/")[1] || "bin";
|
|
18458
|
+
const prefix = contentType === "document" ? "document" : "image";
|
|
18459
|
+
const filename = `${prefix}.${extension}`;
|
|
18460
|
+
const attachment = new Attachment({
|
|
18461
|
+
data: blob,
|
|
18462
|
+
filename,
|
|
18463
|
+
contentType: mediaType
|
|
18464
|
+
});
|
|
18465
|
+
return {
|
|
18466
|
+
...source,
|
|
18467
|
+
data: attachment
|
|
18468
|
+
};
|
|
18469
|
+
}
|
|
18470
|
+
return source;
|
|
18471
|
+
}
|
|
18472
|
+
function processAttachmentsInInput3(input) {
|
|
18473
|
+
if (Array.isArray(input)) {
|
|
18474
|
+
return input.map(processAttachmentsInInput3);
|
|
18475
|
+
}
|
|
18476
|
+
if (isObject(input)) {
|
|
18477
|
+
if ((input.type === "image" || input.type === "document") && isObject(input.source) && input.source.type === "base64") {
|
|
18478
|
+
return {
|
|
18479
|
+
...input,
|
|
18480
|
+
source: convertBase64ToAttachment2(input.source, input.type)
|
|
18481
|
+
};
|
|
18482
|
+
}
|
|
18483
|
+
const processed = {};
|
|
18484
|
+
for (const [key, value] of Object.entries(input)) {
|
|
18485
|
+
processed[key] = processAttachmentsInInput3(value);
|
|
18486
|
+
}
|
|
18487
|
+
return processed;
|
|
18488
|
+
}
|
|
18489
|
+
return input;
|
|
18490
|
+
}
|
|
18491
|
+
function coalesceInput2(messages, system) {
|
|
18492
|
+
const input = (messages || []).slice();
|
|
18493
|
+
if (system) {
|
|
18494
|
+
input.push({ role: "system", content: system });
|
|
18495
|
+
}
|
|
18496
|
+
return input;
|
|
18497
|
+
}
|
|
18498
|
+
function filterFrom2(obj, fieldsToRemove) {
|
|
18499
|
+
const result = {};
|
|
18500
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
18501
|
+
if (!fieldsToRemove.includes(key)) {
|
|
18502
|
+
result[key] = value;
|
|
18503
|
+
}
|
|
18504
|
+
}
|
|
18505
|
+
return result;
|
|
18506
|
+
}
|
|
18507
|
+
|
|
18508
|
+
// src/instrumentation/plugins/ai-sdk-plugin.ts
|
|
18509
|
+
var import_dc_browser3 = require("dc-browser");
|
|
18510
|
+
var DEFAULT_DENY_OUTPUT_PATHS = [
|
|
18511
|
+
// v3
|
|
18512
|
+
"roundtrips[].request.body",
|
|
18513
|
+
"roundtrips[].response.headers",
|
|
18514
|
+
"rawResponse.headers",
|
|
18515
|
+
"responseMessages",
|
|
18516
|
+
// v5
|
|
18517
|
+
"request.body",
|
|
18518
|
+
"response.body",
|
|
18519
|
+
"response.headers",
|
|
18520
|
+
"steps[].request.body",
|
|
18521
|
+
"steps[].response.body",
|
|
18522
|
+
"steps[].response.headers"
|
|
18523
|
+
];
|
|
18524
|
+
var AISDKPlugin = class extends BasePlugin {
|
|
18525
|
+
unsubscribers = [];
|
|
18526
|
+
config;
|
|
18527
|
+
constructor(config = {}) {
|
|
18528
|
+
super();
|
|
18529
|
+
this.config = config;
|
|
18530
|
+
}
|
|
18531
|
+
onEnable() {
|
|
18532
|
+
this.subscribeToAISDK();
|
|
18533
|
+
}
|
|
18534
|
+
onDisable() {
|
|
18535
|
+
for (const unsubscribe of this.unsubscribers) {
|
|
18536
|
+
unsubscribe();
|
|
18537
|
+
}
|
|
18538
|
+
this.unsubscribers = [];
|
|
18539
|
+
}
|
|
18540
|
+
subscribeToAISDK() {
|
|
18541
|
+
const denyOutputPaths = this.config.denyOutputPaths || DEFAULT_DENY_OUTPUT_PATHS;
|
|
18542
|
+
this.subscribeToStreamingChannel("orchestrion:ai-sdk:generateText", {
|
|
18543
|
+
name: "generateText",
|
|
18544
|
+
type: "llm" /* LLM */,
|
|
18545
|
+
extractInput: (args) => {
|
|
18546
|
+
const params = args[0] || {};
|
|
18547
|
+
return {
|
|
18548
|
+
input: processAISDKInput(params),
|
|
18549
|
+
metadata: extractMetadataFromParams(params)
|
|
18550
|
+
};
|
|
18551
|
+
},
|
|
18552
|
+
extractOutput: (result) => {
|
|
18553
|
+
return processAISDKOutput(result, denyOutputPaths);
|
|
18554
|
+
},
|
|
18555
|
+
extractMetrics: (result, startTime) => {
|
|
18556
|
+
const metrics = extractTokenMetrics2(result);
|
|
18557
|
+
if (startTime) {
|
|
18558
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18559
|
+
}
|
|
18560
|
+
return metrics;
|
|
18561
|
+
},
|
|
18562
|
+
aggregateChunks: aggregateAISDKChunks
|
|
18563
|
+
});
|
|
18564
|
+
this.subscribeToStreamingChannel("orchestrion:ai-sdk:streamText", {
|
|
18565
|
+
name: "streamText",
|
|
18566
|
+
type: "llm" /* LLM */,
|
|
18567
|
+
extractInput: (args) => {
|
|
18568
|
+
const params = args[0] || {};
|
|
18569
|
+
return {
|
|
18570
|
+
input: processAISDKInput(params),
|
|
18571
|
+
metadata: extractMetadataFromParams(params)
|
|
18572
|
+
};
|
|
18573
|
+
},
|
|
18574
|
+
extractOutput: (result) => {
|
|
18575
|
+
return processAISDKOutput(result, denyOutputPaths);
|
|
18576
|
+
},
|
|
18577
|
+
extractMetrics: (result, startTime) => {
|
|
18578
|
+
const metrics = extractTokenMetrics2(result);
|
|
18579
|
+
if (startTime) {
|
|
18580
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18581
|
+
}
|
|
18582
|
+
return metrics;
|
|
18583
|
+
},
|
|
18584
|
+
aggregateChunks: aggregateAISDKChunks
|
|
18585
|
+
});
|
|
18586
|
+
this.subscribeToStreamingChannel("orchestrion:ai-sdk:generateObject", {
|
|
18587
|
+
name: "generateObject",
|
|
18588
|
+
type: "llm" /* LLM */,
|
|
18589
|
+
extractInput: (args) => {
|
|
18590
|
+
const params = args[0] || {};
|
|
18591
|
+
return {
|
|
18592
|
+
input: processAISDKInput(params),
|
|
18593
|
+
metadata: extractMetadataFromParams(params)
|
|
18594
|
+
};
|
|
18595
|
+
},
|
|
18596
|
+
extractOutput: (result) => {
|
|
18597
|
+
return processAISDKOutput(result, denyOutputPaths);
|
|
18598
|
+
},
|
|
18599
|
+
extractMetrics: (result, startTime) => {
|
|
18600
|
+
const metrics = extractTokenMetrics2(result);
|
|
18601
|
+
if (startTime) {
|
|
18602
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18603
|
+
}
|
|
18604
|
+
return metrics;
|
|
18605
|
+
},
|
|
18606
|
+
aggregateChunks: aggregateAISDKChunks
|
|
18607
|
+
});
|
|
18608
|
+
this.subscribeToStreamingChannel("orchestrion:ai-sdk:streamObject", {
|
|
18609
|
+
name: "streamObject",
|
|
18610
|
+
type: "llm" /* LLM */,
|
|
18611
|
+
extractInput: (args) => {
|
|
18612
|
+
const params = args[0] || {};
|
|
18613
|
+
return {
|
|
18614
|
+
input: processAISDKInput(params),
|
|
18615
|
+
metadata: extractMetadataFromParams(params)
|
|
18616
|
+
};
|
|
18617
|
+
},
|
|
18618
|
+
extractOutput: (result) => {
|
|
18619
|
+
return processAISDKOutput(result, denyOutputPaths);
|
|
18620
|
+
},
|
|
18621
|
+
extractMetrics: (result, startTime) => {
|
|
18622
|
+
const metrics = extractTokenMetrics2(result);
|
|
18623
|
+
if (startTime) {
|
|
18624
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18625
|
+
}
|
|
18626
|
+
return metrics;
|
|
18627
|
+
},
|
|
18628
|
+
aggregateChunks: aggregateAISDKChunks
|
|
18629
|
+
});
|
|
18630
|
+
this.subscribeToStreamingChannel("orchestrion:ai-sdk:Agent.generate", {
|
|
18631
|
+
name: "Agent.generate",
|
|
18632
|
+
type: "llm" /* LLM */,
|
|
18633
|
+
extractInput: (args) => {
|
|
18634
|
+
const params = args[0] || {};
|
|
18635
|
+
return {
|
|
18636
|
+
input: processAISDKInput(params),
|
|
18637
|
+
metadata: extractMetadataFromParams(params)
|
|
18638
|
+
};
|
|
18639
|
+
},
|
|
18640
|
+
extractOutput: (result) => {
|
|
18641
|
+
return processAISDKOutput(result, denyOutputPaths);
|
|
18642
|
+
},
|
|
18643
|
+
extractMetrics: (result, startTime) => {
|
|
18644
|
+
const metrics = extractTokenMetrics2(result);
|
|
18645
|
+
if (startTime) {
|
|
18646
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18647
|
+
}
|
|
18648
|
+
return metrics;
|
|
18649
|
+
},
|
|
18650
|
+
aggregateChunks: aggregateAISDKChunks
|
|
18651
|
+
});
|
|
18652
|
+
this.subscribeToStreamingChannel("orchestrion:ai-sdk:Agent.stream", {
|
|
18653
|
+
name: "Agent.stream",
|
|
18654
|
+
type: "llm" /* LLM */,
|
|
18655
|
+
extractInput: (args) => {
|
|
18656
|
+
const params = args[0] || {};
|
|
18657
|
+
return {
|
|
18658
|
+
input: processAISDKInput(params),
|
|
18659
|
+
metadata: extractMetadataFromParams(params)
|
|
18660
|
+
};
|
|
18661
|
+
},
|
|
18662
|
+
extractOutput: (result) => {
|
|
18663
|
+
return processAISDKOutput(result, denyOutputPaths);
|
|
18664
|
+
},
|
|
18665
|
+
extractMetrics: (result, startTime) => {
|
|
18666
|
+
const metrics = extractTokenMetrics2(result);
|
|
18667
|
+
if (startTime) {
|
|
18668
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18669
|
+
}
|
|
18670
|
+
return metrics;
|
|
18671
|
+
},
|
|
18672
|
+
aggregateChunks: aggregateAISDKChunks
|
|
18673
|
+
});
|
|
18674
|
+
}
|
|
18675
|
+
/**
|
|
18676
|
+
* Subscribe to a channel for async methods that may return streams.
|
|
18677
|
+
* Handles both streaming and non-streaming responses.
|
|
18678
|
+
*/
|
|
18679
|
+
subscribeToStreamingChannel(channelName, config) {
|
|
18680
|
+
const channel = (0, import_dc_browser3.tracingChannel)(channelName);
|
|
18681
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
18682
|
+
const handlers = {
|
|
18683
|
+
start: (event) => {
|
|
18684
|
+
const span = startSpan({
|
|
18685
|
+
name: config.name,
|
|
18686
|
+
spanAttributes: {
|
|
18687
|
+
type: config.type
|
|
18688
|
+
}
|
|
18689
|
+
});
|
|
18690
|
+
const startTime = getCurrentUnixTimestamp();
|
|
18691
|
+
spans.set(event, { span, startTime });
|
|
18692
|
+
try {
|
|
18693
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
18694
|
+
span.log({
|
|
18695
|
+
input,
|
|
18696
|
+
metadata
|
|
18697
|
+
});
|
|
18698
|
+
} catch (error) {
|
|
18699
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
18700
|
+
}
|
|
18701
|
+
},
|
|
18702
|
+
asyncEnd: (event) => {
|
|
18703
|
+
const spanData = spans.get(event);
|
|
18704
|
+
if (!spanData) {
|
|
18705
|
+
return;
|
|
18706
|
+
}
|
|
18707
|
+
const { span, startTime } = spanData;
|
|
18708
|
+
if (isAsyncIterable4(event.result)) {
|
|
18709
|
+
patchStreamIfNeeded(event.result, {
|
|
18710
|
+
onComplete: (chunks) => {
|
|
18711
|
+
try {
|
|
18712
|
+
let output;
|
|
18713
|
+
let metrics;
|
|
18714
|
+
if (config.aggregateChunks) {
|
|
18715
|
+
const aggregated = config.aggregateChunks(chunks);
|
|
18716
|
+
output = aggregated.output;
|
|
18717
|
+
metrics = aggregated.metrics;
|
|
18718
|
+
} else {
|
|
18719
|
+
output = config.extractOutput(chunks);
|
|
18720
|
+
metrics = config.extractMetrics(chunks, startTime);
|
|
18721
|
+
}
|
|
18722
|
+
if (!metrics.time_to_first_token && chunks.length > 0) {
|
|
18723
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18724
|
+
}
|
|
18725
|
+
span.log({
|
|
18726
|
+
output,
|
|
18727
|
+
metrics
|
|
18728
|
+
});
|
|
18729
|
+
} catch (error) {
|
|
18730
|
+
console.error(
|
|
18731
|
+
`Error extracting output for ${channelName}:`,
|
|
18732
|
+
error
|
|
18733
|
+
);
|
|
18734
|
+
} finally {
|
|
18735
|
+
span.end();
|
|
18736
|
+
}
|
|
18737
|
+
},
|
|
18738
|
+
onError: (error) => {
|
|
18739
|
+
span.log({
|
|
18740
|
+
error: error.message
|
|
18741
|
+
});
|
|
18742
|
+
span.end();
|
|
18743
|
+
}
|
|
18744
|
+
});
|
|
18745
|
+
} else {
|
|
18746
|
+
try {
|
|
18747
|
+
const output = config.extractOutput(event.result);
|
|
18748
|
+
const metrics = config.extractMetrics(event.result, startTime);
|
|
18749
|
+
span.log({
|
|
18750
|
+
output,
|
|
18751
|
+
metrics
|
|
18752
|
+
});
|
|
18753
|
+
} catch (error) {
|
|
18754
|
+
console.error(`Error extracting output for ${channelName}:`, error);
|
|
18755
|
+
} finally {
|
|
18756
|
+
span.end();
|
|
18757
|
+
spans.delete(event);
|
|
18758
|
+
}
|
|
18759
|
+
}
|
|
18760
|
+
},
|
|
18761
|
+
error: (event) => {
|
|
18762
|
+
const spanData = spans.get(event);
|
|
18763
|
+
if (!spanData) {
|
|
18764
|
+
return;
|
|
18765
|
+
}
|
|
18766
|
+
const { span } = spanData;
|
|
18767
|
+
span.log({
|
|
18768
|
+
error: event.error.message
|
|
18769
|
+
});
|
|
18770
|
+
span.end();
|
|
18771
|
+
spans.delete(event);
|
|
18772
|
+
}
|
|
18773
|
+
};
|
|
18774
|
+
channel.subscribe(handlers);
|
|
18775
|
+
this.unsubscribers.push(() => {
|
|
18776
|
+
channel.unsubscribe(handlers);
|
|
18777
|
+
});
|
|
18778
|
+
}
|
|
18779
|
+
};
|
|
18780
|
+
function processAISDKInput(params) {
|
|
18781
|
+
if (!params) return params;
|
|
18782
|
+
return processInputAttachments(params);
|
|
18783
|
+
}
|
|
18784
|
+
function extractMetadataFromParams(params) {
|
|
18785
|
+
const metadata = {
|
|
18786
|
+
braintrust: {
|
|
18787
|
+
integration_name: "ai-sdk",
|
|
18788
|
+
sdk_language: "typescript"
|
|
18789
|
+
}
|
|
18790
|
+
};
|
|
18791
|
+
const { model, provider } = serializeModelWithProvider2(params.model);
|
|
18792
|
+
if (model) {
|
|
18793
|
+
metadata.model = model;
|
|
18794
|
+
}
|
|
18795
|
+
if (provider) {
|
|
18796
|
+
metadata.provider = provider;
|
|
18797
|
+
}
|
|
18798
|
+
return metadata;
|
|
18799
|
+
}
|
|
18800
|
+
function processAISDKOutput(output, denyOutputPaths) {
|
|
18801
|
+
if (!output) return output;
|
|
18802
|
+
const getterValues = extractGetterValues2(output);
|
|
18803
|
+
const merged = { ...output, ...getterValues };
|
|
18804
|
+
return omit2(merged, denyOutputPaths);
|
|
18805
|
+
}
|
|
18806
|
+
function extractTokenMetrics2(result) {
|
|
18807
|
+
const metrics = {};
|
|
18808
|
+
let usage = result?.totalUsage || result?.usage;
|
|
18809
|
+
if (!usage && result) {
|
|
18810
|
+
try {
|
|
18811
|
+
if ("totalUsage" in result && typeof result.totalUsage !== "function") {
|
|
18812
|
+
usage = result.totalUsage;
|
|
18813
|
+
} else if ("usage" in result && typeof result.usage !== "function") {
|
|
18814
|
+
usage = result.usage;
|
|
18815
|
+
}
|
|
18816
|
+
} catch {
|
|
18817
|
+
}
|
|
18818
|
+
}
|
|
18819
|
+
if (!usage) {
|
|
18820
|
+
return metrics;
|
|
18821
|
+
}
|
|
18822
|
+
const promptTokens = firstNumber2(
|
|
18823
|
+
usage.inputTokens?.total,
|
|
18824
|
+
usage.inputTokens,
|
|
18825
|
+
usage.promptTokens,
|
|
18826
|
+
usage.prompt_tokens
|
|
18827
|
+
);
|
|
18828
|
+
if (promptTokens !== void 0) {
|
|
18829
|
+
metrics.prompt_tokens = promptTokens;
|
|
18830
|
+
}
|
|
18831
|
+
const completionTokens = firstNumber2(
|
|
18832
|
+
usage.outputTokens?.total,
|
|
18833
|
+
usage.outputTokens,
|
|
18834
|
+
usage.completionTokens,
|
|
18835
|
+
usage.completion_tokens
|
|
18836
|
+
);
|
|
18837
|
+
if (completionTokens !== void 0) {
|
|
18838
|
+
metrics.completion_tokens = completionTokens;
|
|
18839
|
+
}
|
|
18840
|
+
const totalTokens = firstNumber2(
|
|
18841
|
+
usage.totalTokens,
|
|
18842
|
+
usage.tokens,
|
|
18843
|
+
usage.total_tokens
|
|
18844
|
+
);
|
|
18845
|
+
if (totalTokens !== void 0) {
|
|
18846
|
+
metrics.tokens = totalTokens;
|
|
18847
|
+
}
|
|
18848
|
+
const cost = extractCostFromResult2(result);
|
|
18849
|
+
if (cost !== void 0) {
|
|
18850
|
+
metrics.estimated_cost = cost;
|
|
18851
|
+
}
|
|
18852
|
+
return metrics;
|
|
18853
|
+
}
|
|
18854
|
+
function aggregateAISDKChunks(chunks) {
|
|
18855
|
+
const lastChunk = chunks[chunks.length - 1];
|
|
18856
|
+
const output = {};
|
|
18857
|
+
let metrics = {};
|
|
18858
|
+
if (lastChunk) {
|
|
18859
|
+
metrics = extractTokenMetrics2(lastChunk);
|
|
18860
|
+
if (lastChunk.text !== void 0) {
|
|
18861
|
+
output.text = lastChunk.text;
|
|
18862
|
+
}
|
|
18863
|
+
if (lastChunk.object !== void 0) {
|
|
18864
|
+
output.object = lastChunk.object;
|
|
18865
|
+
}
|
|
18866
|
+
if (lastChunk.finishReason !== void 0) {
|
|
18867
|
+
output.finishReason = lastChunk.finishReason;
|
|
18868
|
+
}
|
|
18869
|
+
if (lastChunk.toolCalls !== void 0) {
|
|
18870
|
+
output.toolCalls = lastChunk.toolCalls;
|
|
18871
|
+
}
|
|
18872
|
+
}
|
|
18873
|
+
return { output, metrics };
|
|
18874
|
+
}
|
|
18875
|
+
function extractGetterValues2(obj) {
|
|
18876
|
+
const getterValues = {};
|
|
18877
|
+
const getterNames = [
|
|
18878
|
+
"text",
|
|
18879
|
+
"object",
|
|
18880
|
+
"finishReason",
|
|
18881
|
+
"usage",
|
|
18882
|
+
"totalUsage",
|
|
18883
|
+
"toolCalls",
|
|
18884
|
+
"toolResults",
|
|
18885
|
+
"warnings",
|
|
18886
|
+
"experimental_providerMetadata",
|
|
18887
|
+
"providerMetadata",
|
|
18888
|
+
"rawResponse",
|
|
18889
|
+
"response"
|
|
18890
|
+
];
|
|
18891
|
+
for (const name of getterNames) {
|
|
18892
|
+
try {
|
|
18893
|
+
if (obj && name in obj && typeof obj[name] !== "function") {
|
|
18894
|
+
getterValues[name] = obj[name];
|
|
18895
|
+
}
|
|
18896
|
+
} catch {
|
|
18897
|
+
}
|
|
18898
|
+
}
|
|
18899
|
+
return getterValues;
|
|
18900
|
+
}
|
|
18901
|
+
function serializeModelWithProvider2(model) {
|
|
18902
|
+
const modelId = typeof model === "string" ? model : model?.modelId;
|
|
18903
|
+
const explicitProvider = typeof model === "object" ? model?.provider : void 0;
|
|
18904
|
+
if (!modelId) {
|
|
18905
|
+
return { model: modelId, provider: explicitProvider };
|
|
18906
|
+
}
|
|
18907
|
+
const parsed = parseGatewayModelString2(modelId);
|
|
18908
|
+
return {
|
|
18909
|
+
model: parsed.model,
|
|
18910
|
+
provider: explicitProvider || parsed.provider
|
|
18911
|
+
};
|
|
18912
|
+
}
|
|
18913
|
+
function parseGatewayModelString2(modelString) {
|
|
18914
|
+
if (!modelString || typeof modelString !== "string") {
|
|
18915
|
+
return { model: modelString };
|
|
18916
|
+
}
|
|
18917
|
+
const slashIndex = modelString.indexOf("/");
|
|
18918
|
+
if (slashIndex > 0 && slashIndex < modelString.length - 1) {
|
|
18919
|
+
return {
|
|
18920
|
+
provider: modelString.substring(0, slashIndex),
|
|
18921
|
+
model: modelString.substring(slashIndex + 1)
|
|
18922
|
+
};
|
|
18923
|
+
}
|
|
18924
|
+
return { model: modelString };
|
|
18925
|
+
}
|
|
18926
|
+
function extractCostFromResult2(result) {
|
|
18927
|
+
if (result?.steps && Array.isArray(result.steps) && result.steps.length > 0) {
|
|
18928
|
+
let totalCost = 0;
|
|
18929
|
+
let foundCost = false;
|
|
18930
|
+
for (const step of result.steps) {
|
|
18931
|
+
const gateway2 = step?.providerMetadata?.gateway;
|
|
18932
|
+
const stepCost = parseGatewayCost2(gateway2?.cost) || parseGatewayCost2(gateway2?.marketCost);
|
|
18933
|
+
if (stepCost !== void 0 && stepCost > 0) {
|
|
18934
|
+
totalCost += stepCost;
|
|
18935
|
+
foundCost = true;
|
|
18936
|
+
}
|
|
18937
|
+
}
|
|
18938
|
+
if (foundCost) {
|
|
18939
|
+
return totalCost;
|
|
18940
|
+
}
|
|
18941
|
+
}
|
|
18942
|
+
const gateway = result?.providerMetadata?.gateway;
|
|
18943
|
+
const directCost = parseGatewayCost2(gateway?.cost) || parseGatewayCost2(gateway?.marketCost);
|
|
18944
|
+
if (directCost !== void 0 && directCost > 0) {
|
|
18945
|
+
return directCost;
|
|
18946
|
+
}
|
|
18947
|
+
return void 0;
|
|
18948
|
+
}
|
|
18949
|
+
function parseGatewayCost2(cost) {
|
|
18950
|
+
if (cost === void 0 || cost === null) {
|
|
18951
|
+
return void 0;
|
|
18952
|
+
}
|
|
18953
|
+
if (typeof cost === "number") {
|
|
18954
|
+
return cost;
|
|
18955
|
+
}
|
|
18956
|
+
if (typeof cost === "string") {
|
|
18957
|
+
const parsed = parseFloat(cost);
|
|
18958
|
+
if (!isNaN(parsed)) {
|
|
18959
|
+
return parsed;
|
|
18960
|
+
}
|
|
18961
|
+
}
|
|
18962
|
+
return void 0;
|
|
18963
|
+
}
|
|
18964
|
+
function firstNumber2(...values) {
|
|
18965
|
+
for (const v of values) {
|
|
18966
|
+
if (typeof v === "number") {
|
|
18967
|
+
return v;
|
|
18968
|
+
}
|
|
18969
|
+
}
|
|
18970
|
+
return void 0;
|
|
18971
|
+
}
|
|
18972
|
+
function deepCopy2(obj) {
|
|
18973
|
+
return JSON.parse(JSON.stringify(obj));
|
|
18974
|
+
}
|
|
18975
|
+
function parsePath2(path) {
|
|
18976
|
+
const keys = [];
|
|
18977
|
+
let current = "";
|
|
18978
|
+
for (let i = 0; i < path.length; i++) {
|
|
18979
|
+
const char = path[i];
|
|
18980
|
+
if (char === ".") {
|
|
18981
|
+
if (current) {
|
|
18982
|
+
keys.push(current);
|
|
18983
|
+
current = "";
|
|
18984
|
+
}
|
|
18985
|
+
} else if (char === "[") {
|
|
18986
|
+
if (current) {
|
|
18987
|
+
keys.push(current);
|
|
18988
|
+
current = "";
|
|
18989
|
+
}
|
|
18990
|
+
let bracketContent = "";
|
|
18991
|
+
i++;
|
|
18992
|
+
while (i < path.length && path[i] !== "]") {
|
|
18993
|
+
bracketContent += path[i];
|
|
18994
|
+
i++;
|
|
18995
|
+
}
|
|
18996
|
+
if (bracketContent === "") {
|
|
18997
|
+
keys.push("[]");
|
|
18998
|
+
} else {
|
|
18999
|
+
const index = parseInt(bracketContent, 10);
|
|
19000
|
+
keys.push(isNaN(index) ? bracketContent : index);
|
|
19001
|
+
}
|
|
19002
|
+
} else {
|
|
19003
|
+
current += char;
|
|
19004
|
+
}
|
|
19005
|
+
}
|
|
19006
|
+
if (current) {
|
|
19007
|
+
keys.push(current);
|
|
19008
|
+
}
|
|
19009
|
+
return keys;
|
|
19010
|
+
}
|
|
19011
|
+
function omitAtPath2(obj, keys) {
|
|
19012
|
+
if (keys.length === 0) return;
|
|
19013
|
+
const firstKey = keys[0];
|
|
19014
|
+
const remainingKeys = keys.slice(1);
|
|
19015
|
+
if (firstKey === "[]") {
|
|
19016
|
+
if (Array.isArray(obj)) {
|
|
19017
|
+
obj.forEach((item) => {
|
|
19018
|
+
if (remainingKeys.length > 0) {
|
|
19019
|
+
omitAtPath2(item, remainingKeys);
|
|
19020
|
+
}
|
|
19021
|
+
});
|
|
19022
|
+
}
|
|
19023
|
+
} else if (remainingKeys.length === 0) {
|
|
19024
|
+
if (obj && typeof obj === "object" && firstKey in obj) {
|
|
19025
|
+
obj[firstKey] = "<omitted>";
|
|
19026
|
+
}
|
|
19027
|
+
} else {
|
|
19028
|
+
if (obj && typeof obj === "object" && firstKey in obj) {
|
|
19029
|
+
omitAtPath2(obj[firstKey], remainingKeys);
|
|
19030
|
+
}
|
|
19031
|
+
}
|
|
19032
|
+
}
|
|
19033
|
+
function omit2(obj, paths) {
|
|
19034
|
+
const result = deepCopy2(obj);
|
|
19035
|
+
for (const path of paths) {
|
|
19036
|
+
const keys = parsePath2(path);
|
|
19037
|
+
omitAtPath2(result, keys);
|
|
19038
|
+
}
|
|
19039
|
+
return result;
|
|
19040
|
+
}
|
|
19041
|
+
|
|
19042
|
+
// src/instrumentation/plugins/claude-agent-sdk-plugin.ts
|
|
19043
|
+
var import_dc_browser4 = require("dc-browser");
|
|
19044
|
+
function filterSerializableOptions2(options) {
|
|
19045
|
+
const allowedKeys = [
|
|
19046
|
+
"model",
|
|
19047
|
+
"maxTurns",
|
|
19048
|
+
"cwd",
|
|
19049
|
+
"continue",
|
|
19050
|
+
"allowedTools",
|
|
19051
|
+
"disallowedTools",
|
|
19052
|
+
"additionalDirectories",
|
|
19053
|
+
"permissionMode",
|
|
19054
|
+
"debug",
|
|
19055
|
+
"apiKey",
|
|
19056
|
+
"apiKeySource",
|
|
19057
|
+
"agentName",
|
|
19058
|
+
"instructions"
|
|
19059
|
+
];
|
|
19060
|
+
const filtered = {};
|
|
19061
|
+
for (const key of allowedKeys) {
|
|
19062
|
+
if (options[key] !== void 0) {
|
|
19063
|
+
filtered[key] = options[key];
|
|
19064
|
+
}
|
|
19065
|
+
}
|
|
19066
|
+
return filtered;
|
|
19067
|
+
}
|
|
19068
|
+
function getNumberProperty3(obj, key) {
|
|
19069
|
+
if (!obj || typeof obj !== "object" || !(key in obj)) {
|
|
19070
|
+
return void 0;
|
|
19071
|
+
}
|
|
19072
|
+
const value = Reflect.get(obj, key);
|
|
19073
|
+
return typeof value === "number" ? value : void 0;
|
|
19074
|
+
}
|
|
19075
|
+
function extractUsageFromMessage(message) {
|
|
19076
|
+
const metrics = {};
|
|
19077
|
+
let usage;
|
|
19078
|
+
if (message.type === "assistant") {
|
|
19079
|
+
usage = message.message?.usage;
|
|
19080
|
+
} else if (message.type === "result") {
|
|
19081
|
+
usage = message.usage;
|
|
19082
|
+
}
|
|
19083
|
+
if (!usage || typeof usage !== "object") {
|
|
19084
|
+
return metrics;
|
|
19085
|
+
}
|
|
19086
|
+
const inputTokens = getNumberProperty3(usage, "input_tokens");
|
|
19087
|
+
if (inputTokens !== void 0) {
|
|
19088
|
+
metrics.prompt_tokens = inputTokens;
|
|
19089
|
+
}
|
|
19090
|
+
const outputTokens = getNumberProperty3(usage, "output_tokens");
|
|
19091
|
+
if (outputTokens !== void 0) {
|
|
19092
|
+
metrics.completion_tokens = outputTokens;
|
|
19093
|
+
}
|
|
19094
|
+
const cacheReadTokens = getNumberProperty3(usage, "cache_read_input_tokens") || 0;
|
|
19095
|
+
const cacheCreationTokens = getNumberProperty3(usage, "cache_creation_input_tokens") || 0;
|
|
19096
|
+
if (cacheReadTokens > 0 || cacheCreationTokens > 0) {
|
|
19097
|
+
const cacheTokens = extractAnthropicCacheTokens(
|
|
19098
|
+
cacheReadTokens,
|
|
19099
|
+
cacheCreationTokens
|
|
19100
|
+
);
|
|
19101
|
+
Object.assign(metrics, cacheTokens);
|
|
19102
|
+
}
|
|
19103
|
+
if (Object.keys(metrics).length > 0) {
|
|
19104
|
+
Object.assign(metrics, finalizeAnthropicTokens(metrics));
|
|
19105
|
+
}
|
|
19106
|
+
return metrics;
|
|
19107
|
+
}
|
|
19108
|
+
function buildLLMInput(prompt, conversationHistory) {
|
|
19109
|
+
const promptMessage = typeof prompt === "string" ? { content: prompt, role: "user" } : void 0;
|
|
19110
|
+
const inputParts = [
|
|
19111
|
+
...promptMessage ? [promptMessage] : [],
|
|
19112
|
+
...conversationHistory
|
|
19113
|
+
];
|
|
19114
|
+
return inputParts.length > 0 ? inputParts : void 0;
|
|
19115
|
+
}
|
|
19116
|
+
async function createLLMSpanForMessages(messages, prompt, conversationHistory, options, startTime, parentSpan) {
|
|
19117
|
+
if (messages.length === 0) return void 0;
|
|
19118
|
+
const lastMessage = messages[messages.length - 1];
|
|
19119
|
+
if (lastMessage.type !== "assistant" || !lastMessage.message?.usage) {
|
|
19120
|
+
return void 0;
|
|
19121
|
+
}
|
|
19122
|
+
const model = lastMessage.message.model || options.model;
|
|
19123
|
+
const usage = extractUsageFromMessage(lastMessage);
|
|
19124
|
+
const input = buildLLMInput(prompt, conversationHistory);
|
|
19125
|
+
const outputs = messages.map(
|
|
19126
|
+
(m) => m.message?.content && m.message?.role ? { content: m.message.content, role: m.message.role } : void 0
|
|
19127
|
+
).filter((c) => c !== void 0);
|
|
19128
|
+
const span = startSpan({
|
|
19129
|
+
name: "anthropic.messages.create",
|
|
19130
|
+
spanAttributes: {
|
|
19131
|
+
type: "llm" /* LLM */
|
|
19132
|
+
},
|
|
19133
|
+
startTime,
|
|
19134
|
+
parent: parentSpan
|
|
19135
|
+
});
|
|
19136
|
+
span.log({
|
|
19137
|
+
input,
|
|
19138
|
+
output: outputs,
|
|
19139
|
+
metadata: model ? { model } : void 0,
|
|
19140
|
+
metrics: usage
|
|
19141
|
+
});
|
|
19142
|
+
await span.end();
|
|
19143
|
+
return lastMessage.message?.content && lastMessage.message?.role ? { content: lastMessage.message.content, role: lastMessage.message.role } : void 0;
|
|
19144
|
+
}
|
|
19145
|
+
var ClaudeAgentSDKPlugin = class extends BasePlugin {
|
|
19146
|
+
unsubscribers = [];
|
|
19147
|
+
onEnable() {
|
|
19148
|
+
this.subscribeToQuery();
|
|
19149
|
+
}
|
|
19150
|
+
onDisable() {
|
|
19151
|
+
for (const unsubscribe of this.unsubscribers) {
|
|
19152
|
+
unsubscribe();
|
|
19153
|
+
}
|
|
19154
|
+
this.unsubscribers = [];
|
|
19155
|
+
}
|
|
19156
|
+
/**
|
|
19157
|
+
* Subscribe to the query channel for agent interactions.
|
|
19158
|
+
* Handles streaming responses and traces both the top-level agent task
|
|
19159
|
+
* and individual LLM calls.
|
|
19160
|
+
*/
|
|
19161
|
+
subscribeToQuery() {
|
|
19162
|
+
const channel = (0, import_dc_browser4.tracingChannel)("orchestrion:claude-agent-sdk:query");
|
|
19163
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
19164
|
+
const handlers = {
|
|
19165
|
+
start: (event) => {
|
|
19166
|
+
const params = event.arguments[0] ?? {};
|
|
19167
|
+
const { prompt, options = {} } = params;
|
|
19168
|
+
const span = startSpan({
|
|
19169
|
+
name: "Claude Agent",
|
|
19170
|
+
spanAttributes: {
|
|
19171
|
+
type: "task" /* TASK */
|
|
19172
|
+
}
|
|
19173
|
+
});
|
|
19174
|
+
const startTime = getCurrentUnixTimestamp();
|
|
19175
|
+
try {
|
|
19176
|
+
span.log({
|
|
19177
|
+
input: typeof prompt === "string" ? prompt : {
|
|
19178
|
+
type: "streaming",
|
|
19179
|
+
description: "AsyncIterable<SDKMessage>"
|
|
19180
|
+
},
|
|
19181
|
+
metadata: filterSerializableOptions2(options)
|
|
19182
|
+
});
|
|
19183
|
+
} catch (error) {
|
|
19184
|
+
console.error("Error extracting input for Claude Agent SDK:", error);
|
|
19185
|
+
}
|
|
19186
|
+
spans.set(event, {
|
|
19187
|
+
span,
|
|
19188
|
+
startTime,
|
|
19189
|
+
conversationHistory: [],
|
|
19190
|
+
currentMessages: [],
|
|
19191
|
+
currentMessageId: void 0,
|
|
19192
|
+
currentMessageStartTime: startTime,
|
|
19193
|
+
accumulatedOutputTokens: 0
|
|
19194
|
+
});
|
|
19195
|
+
},
|
|
19196
|
+
asyncEnd: (event) => {
|
|
19197
|
+
const spanData = spans.get(event);
|
|
19198
|
+
if (!spanData) {
|
|
19199
|
+
return;
|
|
19200
|
+
}
|
|
19201
|
+
if (isAsyncIterable4(event.result)) {
|
|
19202
|
+
patchStreamIfNeeded(event.result, {
|
|
19203
|
+
onChunk: async (message) => {
|
|
19204
|
+
const currentTime = getCurrentUnixTimestamp();
|
|
19205
|
+
const params = event.arguments[0];
|
|
19206
|
+
const { prompt, options = {} } = params;
|
|
19207
|
+
const messageId = message.message?.id;
|
|
19208
|
+
if (messageId && messageId !== spanData.currentMessageId) {
|
|
19209
|
+
if (spanData.currentMessages.length > 0) {
|
|
19210
|
+
const finalMessage = await createLLMSpanForMessages(
|
|
19211
|
+
spanData.currentMessages,
|
|
19212
|
+
prompt,
|
|
19213
|
+
spanData.conversationHistory,
|
|
19214
|
+
options,
|
|
19215
|
+
spanData.currentMessageStartTime,
|
|
19216
|
+
await spanData.span.export()
|
|
19217
|
+
);
|
|
19218
|
+
if (finalMessage) {
|
|
19219
|
+
spanData.conversationHistory.push(finalMessage);
|
|
19220
|
+
}
|
|
19221
|
+
const lastMessage = spanData.currentMessages[spanData.currentMessages.length - 1];
|
|
19222
|
+
if (lastMessage?.message?.usage) {
|
|
19223
|
+
const outputTokens = getNumberProperty3(
|
|
19224
|
+
lastMessage.message.usage,
|
|
19225
|
+
"output_tokens"
|
|
19226
|
+
) || 0;
|
|
19227
|
+
spanData.accumulatedOutputTokens += outputTokens;
|
|
19228
|
+
}
|
|
19229
|
+
spanData.currentMessages = [];
|
|
19230
|
+
}
|
|
19231
|
+
spanData.currentMessageId = messageId;
|
|
19232
|
+
spanData.currentMessageStartTime = currentTime;
|
|
19233
|
+
}
|
|
19234
|
+
if (message.type === "assistant" && message.message?.usage) {
|
|
19235
|
+
spanData.currentMessages.push(message);
|
|
19236
|
+
}
|
|
19237
|
+
if (message.type === "result" && message.usage) {
|
|
19238
|
+
const finalUsageMetrics = extractUsageFromMessage(message);
|
|
19239
|
+
if (spanData.currentMessages.length > 0 && finalUsageMetrics.completion_tokens !== void 0) {
|
|
19240
|
+
const lastMessage = spanData.currentMessages[spanData.currentMessages.length - 1];
|
|
19241
|
+
if (lastMessage?.message?.usage) {
|
|
19242
|
+
const adjustedTokens = finalUsageMetrics.completion_tokens - spanData.accumulatedOutputTokens;
|
|
19243
|
+
if (adjustedTokens >= 0) {
|
|
19244
|
+
lastMessage.message.usage.output_tokens = adjustedTokens;
|
|
19245
|
+
}
|
|
19246
|
+
}
|
|
19247
|
+
}
|
|
19248
|
+
const result_metadata = {};
|
|
19249
|
+
if (message.num_turns !== void 0) {
|
|
19250
|
+
result_metadata.num_turns = message.num_turns;
|
|
19251
|
+
}
|
|
19252
|
+
if (message.session_id !== void 0) {
|
|
19253
|
+
result_metadata.session_id = message.session_id;
|
|
19254
|
+
}
|
|
19255
|
+
if (Object.keys(result_metadata).length > 0) {
|
|
19256
|
+
spanData.span.log({
|
|
19257
|
+
metadata: result_metadata
|
|
19258
|
+
});
|
|
19259
|
+
}
|
|
19260
|
+
}
|
|
19261
|
+
},
|
|
19262
|
+
onComplete: async () => {
|
|
19263
|
+
try {
|
|
19264
|
+
const params = event.arguments[0];
|
|
19265
|
+
const { prompt, options = {} } = params;
|
|
19266
|
+
if (spanData.currentMessages.length > 0) {
|
|
19267
|
+
const finalMessage = await createLLMSpanForMessages(
|
|
19268
|
+
spanData.currentMessages,
|
|
19269
|
+
prompt,
|
|
19270
|
+
spanData.conversationHistory,
|
|
19271
|
+
options,
|
|
19272
|
+
spanData.currentMessageStartTime,
|
|
19273
|
+
await spanData.span.export()
|
|
19274
|
+
);
|
|
19275
|
+
if (finalMessage) {
|
|
19276
|
+
spanData.conversationHistory.push(finalMessage);
|
|
19277
|
+
}
|
|
19278
|
+
}
|
|
19279
|
+
spanData.span.log({
|
|
19280
|
+
output: spanData.conversationHistory.length > 0 ? spanData.conversationHistory[spanData.conversationHistory.length - 1] : void 0
|
|
19281
|
+
});
|
|
19282
|
+
} catch (error) {
|
|
19283
|
+
console.error(
|
|
19284
|
+
"Error extracting output for Claude Agent SDK:",
|
|
19285
|
+
error
|
|
19286
|
+
);
|
|
19287
|
+
} finally {
|
|
19288
|
+
spanData.span.end();
|
|
19289
|
+
spans.delete(event);
|
|
19290
|
+
}
|
|
19291
|
+
},
|
|
19292
|
+
onError: (error) => {
|
|
19293
|
+
spanData.span.log({
|
|
19294
|
+
error: error.message
|
|
19295
|
+
});
|
|
19296
|
+
spanData.span.end();
|
|
19297
|
+
spans.delete(event);
|
|
19298
|
+
}
|
|
19299
|
+
});
|
|
19300
|
+
} else {
|
|
19301
|
+
try {
|
|
19302
|
+
spanData.span.log({
|
|
19303
|
+
output: event.result
|
|
19304
|
+
});
|
|
19305
|
+
} catch (error) {
|
|
19306
|
+
console.error(
|
|
19307
|
+
"Error extracting output for Claude Agent SDK:",
|
|
19308
|
+
error
|
|
19309
|
+
);
|
|
19310
|
+
} finally {
|
|
19311
|
+
spanData.span.end();
|
|
19312
|
+
spans.delete(event);
|
|
19313
|
+
}
|
|
19314
|
+
}
|
|
19315
|
+
},
|
|
19316
|
+
error: (event) => {
|
|
19317
|
+
const spanData = spans.get(event);
|
|
19318
|
+
if (!spanData) {
|
|
19319
|
+
return;
|
|
19320
|
+
}
|
|
19321
|
+
const { span } = spanData;
|
|
19322
|
+
span.log({
|
|
19323
|
+
error: event.error.message
|
|
19324
|
+
});
|
|
19325
|
+
span.end();
|
|
19326
|
+
spans.delete(event);
|
|
19327
|
+
}
|
|
19328
|
+
};
|
|
19329
|
+
channel.subscribe(handlers);
|
|
19330
|
+
this.unsubscribers.push(() => {
|
|
19331
|
+
channel.unsubscribe(handlers);
|
|
19332
|
+
});
|
|
19333
|
+
}
|
|
19334
|
+
};
|
|
19335
|
+
|
|
19336
|
+
// src/instrumentation/plugins/google-genai-plugin.ts
|
|
19337
|
+
var import_dc_browser5 = require("dc-browser");
|
|
19338
|
+
var GoogleGenAIPlugin = class extends BasePlugin {
|
|
19339
|
+
unsubscribers = [];
|
|
19340
|
+
onEnable() {
|
|
19341
|
+
this.subscribeToGoogleGenAIChannels();
|
|
19342
|
+
}
|
|
19343
|
+
onDisable() {
|
|
19344
|
+
for (const unsubscribe of this.unsubscribers) {
|
|
19345
|
+
unsubscribe();
|
|
19346
|
+
}
|
|
19347
|
+
this.unsubscribers = [];
|
|
19348
|
+
}
|
|
19349
|
+
subscribeToGoogleGenAIChannels() {
|
|
19350
|
+
this.subscribeToChannel("orchestrion:google-genai:models.generateContent", {
|
|
19351
|
+
name: "google-genai.generateContent",
|
|
19352
|
+
type: "llm" /* LLM */,
|
|
19353
|
+
extractInput: (args) => {
|
|
19354
|
+
const params = args[0] || {};
|
|
19355
|
+
const input = serializeInput2(params);
|
|
19356
|
+
const metadata = extractMetadata2(params);
|
|
19357
|
+
return {
|
|
19358
|
+
input,
|
|
19359
|
+
metadata: { ...metadata, provider: "google-genai" }
|
|
19360
|
+
};
|
|
19361
|
+
},
|
|
19362
|
+
extractOutput: (result) => {
|
|
19363
|
+
return result;
|
|
19364
|
+
},
|
|
19365
|
+
extractMetrics: (result, startTime) => {
|
|
19366
|
+
return extractGenerateContentMetrics2(result, startTime);
|
|
19367
|
+
}
|
|
19368
|
+
});
|
|
19369
|
+
this.subscribeToGoogleStreamingChannel(
|
|
19370
|
+
"orchestrion:google-genai:models.generateContentStream",
|
|
19371
|
+
{
|
|
19372
|
+
name: "google-genai.generateContentStream",
|
|
19373
|
+
type: "llm" /* LLM */,
|
|
19374
|
+
extractInput: (args) => {
|
|
19375
|
+
const params = args[0] || {};
|
|
19376
|
+
const input = serializeInput2(params);
|
|
19377
|
+
const metadata = extractMetadata2(params);
|
|
19378
|
+
return {
|
|
19379
|
+
input,
|
|
19380
|
+
metadata: { ...metadata, provider: "google-genai" }
|
|
19381
|
+
};
|
|
19382
|
+
},
|
|
19383
|
+
aggregateChunks: aggregateGenerateContentChunks2
|
|
19384
|
+
}
|
|
19385
|
+
);
|
|
19386
|
+
}
|
|
19387
|
+
subscribeToChannel(channelName, config) {
|
|
19388
|
+
const channel = (0, import_dc_browser5.tracingChannel)(channelName);
|
|
19389
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
19390
|
+
const handlers = {
|
|
19391
|
+
start: (event) => {
|
|
19392
|
+
const span = startSpan({
|
|
19393
|
+
name: config.name,
|
|
19394
|
+
spanAttributes: {
|
|
19395
|
+
type: config.type
|
|
19396
|
+
}
|
|
19397
|
+
});
|
|
19398
|
+
const startTime = getCurrentUnixTimestamp();
|
|
19399
|
+
spans.set(event, { span, startTime });
|
|
19400
|
+
try {
|
|
19401
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
19402
|
+
span.log({
|
|
19403
|
+
input,
|
|
19404
|
+
metadata
|
|
19405
|
+
});
|
|
19406
|
+
} catch (error) {
|
|
19407
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
19408
|
+
}
|
|
19409
|
+
},
|
|
19410
|
+
asyncEnd: (event) => {
|
|
19411
|
+
const spanData = spans.get(event);
|
|
19412
|
+
if (!spanData) {
|
|
19413
|
+
return;
|
|
19414
|
+
}
|
|
19415
|
+
const { span, startTime } = spanData;
|
|
19416
|
+
try {
|
|
19417
|
+
const output = config.extractOutput(event.result);
|
|
19418
|
+
const metrics = config.extractMetrics(event.result, startTime);
|
|
19419
|
+
span.log({
|
|
19420
|
+
output,
|
|
19421
|
+
metrics
|
|
19422
|
+
});
|
|
19423
|
+
} catch (error) {
|
|
19424
|
+
console.error(`Error extracting output for ${channelName}:`, error);
|
|
19425
|
+
} finally {
|
|
19426
|
+
span.end();
|
|
19427
|
+
spans.delete(event);
|
|
19428
|
+
}
|
|
19429
|
+
},
|
|
19430
|
+
error: (event) => {
|
|
19431
|
+
const spanData = spans.get(event);
|
|
19432
|
+
if (!spanData) {
|
|
19433
|
+
return;
|
|
19434
|
+
}
|
|
19435
|
+
const { span } = spanData;
|
|
19436
|
+
span.log({
|
|
19437
|
+
error: event.error.message
|
|
19438
|
+
});
|
|
19439
|
+
span.end();
|
|
19440
|
+
spans.delete(event);
|
|
19441
|
+
}
|
|
19442
|
+
};
|
|
19443
|
+
channel.subscribe(handlers);
|
|
19444
|
+
this.unsubscribers.push(() => {
|
|
19445
|
+
channel.unsubscribe(handlers);
|
|
19446
|
+
});
|
|
19447
|
+
}
|
|
19448
|
+
subscribeToGoogleStreamingChannel(channelName, config) {
|
|
19449
|
+
const channel = (0, import_dc_browser5.tracingChannel)(channelName);
|
|
19450
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
19451
|
+
const handlers = {
|
|
19452
|
+
start: (event) => {
|
|
19453
|
+
const span = startSpan({
|
|
19454
|
+
name: config.name,
|
|
19455
|
+
spanAttributes: {
|
|
19456
|
+
type: config.type
|
|
19457
|
+
}
|
|
19458
|
+
});
|
|
19459
|
+
const startTime = getCurrentUnixTimestamp();
|
|
19460
|
+
spans.set(event, { span, startTime });
|
|
19461
|
+
try {
|
|
19462
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
19463
|
+
span.log({
|
|
19464
|
+
input,
|
|
19465
|
+
metadata
|
|
19466
|
+
});
|
|
19467
|
+
} catch (error) {
|
|
19468
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
19469
|
+
}
|
|
19470
|
+
},
|
|
19471
|
+
asyncEnd: (event) => {
|
|
19472
|
+
const spanData = spans.get(event);
|
|
19473
|
+
if (!spanData) {
|
|
19474
|
+
return;
|
|
19475
|
+
}
|
|
19476
|
+
const { span, startTime } = spanData;
|
|
19477
|
+
if (isAsyncIterable4(event.result)) {
|
|
19478
|
+
patchStreamIfNeeded(event.result, {
|
|
19479
|
+
onComplete: (chunks) => {
|
|
19480
|
+
try {
|
|
19481
|
+
const { output, metrics } = config.aggregateChunks(
|
|
19482
|
+
chunks,
|
|
19483
|
+
startTime
|
|
19484
|
+
);
|
|
19485
|
+
span.log({
|
|
19486
|
+
output,
|
|
19487
|
+
metrics
|
|
19488
|
+
});
|
|
19489
|
+
} catch (error) {
|
|
19490
|
+
console.error(
|
|
19491
|
+
`Error extracting output for ${channelName}:`,
|
|
19492
|
+
error
|
|
19493
|
+
);
|
|
19494
|
+
} finally {
|
|
19495
|
+
span.end();
|
|
19496
|
+
}
|
|
19497
|
+
},
|
|
19498
|
+
onError: (error) => {
|
|
19499
|
+
span.log({
|
|
19500
|
+
error: error.message
|
|
19501
|
+
});
|
|
19502
|
+
span.end();
|
|
19503
|
+
}
|
|
19504
|
+
});
|
|
19505
|
+
} else {
|
|
19506
|
+
span.end();
|
|
19507
|
+
spans.delete(event);
|
|
19508
|
+
}
|
|
19509
|
+
},
|
|
19510
|
+
error: (event) => {
|
|
19511
|
+
const spanData = spans.get(event);
|
|
19512
|
+
if (!spanData) {
|
|
19513
|
+
return;
|
|
19514
|
+
}
|
|
19515
|
+
const { span } = spanData;
|
|
19516
|
+
span.log({
|
|
19517
|
+
error: event.error.message
|
|
19518
|
+
});
|
|
19519
|
+
span.end();
|
|
19520
|
+
spans.delete(event);
|
|
19521
|
+
}
|
|
19522
|
+
};
|
|
19523
|
+
channel.subscribe(handlers);
|
|
19524
|
+
this.unsubscribers.push(() => {
|
|
19525
|
+
channel.unsubscribe(handlers);
|
|
19526
|
+
});
|
|
19527
|
+
}
|
|
19528
|
+
};
|
|
19529
|
+
function serializeInput2(params) {
|
|
19530
|
+
const input = {
|
|
19531
|
+
model: params.model,
|
|
19532
|
+
contents: serializeContents2(params.contents)
|
|
19533
|
+
};
|
|
19534
|
+
if (params.config) {
|
|
19535
|
+
const config = tryToDict2(params.config);
|
|
19536
|
+
if (config) {
|
|
19537
|
+
const tools = serializeTools2(params);
|
|
19538
|
+
if (tools) {
|
|
19539
|
+
config.tools = tools;
|
|
19540
|
+
}
|
|
19541
|
+
input.config = config;
|
|
19542
|
+
}
|
|
19543
|
+
}
|
|
19544
|
+
return input;
|
|
19545
|
+
}
|
|
19546
|
+
function serializeContents2(contents) {
|
|
19547
|
+
if (contents === null || contents === void 0) {
|
|
19548
|
+
return null;
|
|
19549
|
+
}
|
|
19550
|
+
if (Array.isArray(contents)) {
|
|
19551
|
+
return contents.map((item) => serializeContentItem2(item));
|
|
19552
|
+
}
|
|
19553
|
+
return serializeContentItem2(contents);
|
|
19554
|
+
}
|
|
19555
|
+
function serializeContentItem2(item) {
|
|
19556
|
+
if (typeof item === "object" && item !== null) {
|
|
19557
|
+
if (item.parts && Array.isArray(item.parts)) {
|
|
19558
|
+
return {
|
|
19559
|
+
...item,
|
|
19560
|
+
parts: item.parts.map((part) => serializePart2(part))
|
|
19561
|
+
};
|
|
19562
|
+
}
|
|
19563
|
+
return item;
|
|
19564
|
+
}
|
|
19565
|
+
if (typeof item === "string") {
|
|
19566
|
+
return { text: item };
|
|
19567
|
+
}
|
|
19568
|
+
return item;
|
|
19569
|
+
}
|
|
19570
|
+
function serializePart2(part) {
|
|
19571
|
+
if (!part || typeof part !== "object") {
|
|
19572
|
+
return part;
|
|
19573
|
+
}
|
|
19574
|
+
if (part.inlineData && part.inlineData.data) {
|
|
19575
|
+
const { data, mimeType } = part.inlineData;
|
|
19576
|
+
if (data instanceof Uint8Array || typeof Buffer !== "undefined" && Buffer.isBuffer(data) || typeof data === "string") {
|
|
19577
|
+
const extension = mimeType ? mimeType.split("/")[1] : "bin";
|
|
19578
|
+
const filename = `file.${extension}`;
|
|
19579
|
+
const buffer = typeof data === "string" ? typeof Buffer !== "undefined" ? Buffer.from(data, "base64") : new Uint8Array(
|
|
19580
|
+
atob(data).split("").map((c) => c.charCodeAt(0))
|
|
19581
|
+
) : typeof Buffer !== "undefined" ? Buffer.from(data) : new Uint8Array(data);
|
|
19582
|
+
const attachment = new Attachment({
|
|
19583
|
+
data: buffer,
|
|
19584
|
+
filename,
|
|
19585
|
+
contentType: mimeType || "application/octet-stream"
|
|
19586
|
+
});
|
|
19587
|
+
return {
|
|
19588
|
+
image_url: { url: attachment }
|
|
19589
|
+
};
|
|
19590
|
+
}
|
|
19591
|
+
}
|
|
19592
|
+
return part;
|
|
19593
|
+
}
|
|
19594
|
+
function serializeTools2(params) {
|
|
19595
|
+
if (!params.config?.tools) {
|
|
19596
|
+
return null;
|
|
19597
|
+
}
|
|
19598
|
+
try {
|
|
19599
|
+
return params.config.tools.map((tool) => {
|
|
19600
|
+
if (typeof tool === "object" && tool.functionDeclarations) {
|
|
19601
|
+
return tool;
|
|
19602
|
+
}
|
|
19603
|
+
return tool;
|
|
19604
|
+
});
|
|
19605
|
+
} catch {
|
|
19606
|
+
return null;
|
|
19607
|
+
}
|
|
19608
|
+
}
|
|
19609
|
+
function extractMetadata2(params) {
|
|
19610
|
+
const metadata = {};
|
|
19611
|
+
if (params.model) {
|
|
19612
|
+
metadata.model = params.model;
|
|
19613
|
+
}
|
|
19614
|
+
if (params.config) {
|
|
19615
|
+
const config = tryToDict2(params.config);
|
|
19616
|
+
if (config) {
|
|
19617
|
+
Object.keys(config).forEach((key) => {
|
|
19618
|
+
if (key !== "tools") {
|
|
19619
|
+
metadata[key] = config[key];
|
|
19620
|
+
}
|
|
19621
|
+
});
|
|
19622
|
+
}
|
|
19623
|
+
}
|
|
19624
|
+
return metadata;
|
|
19625
|
+
}
|
|
19626
|
+
function extractGenerateContentMetrics2(response, startTime) {
|
|
19627
|
+
const metrics = {};
|
|
19628
|
+
if (startTime) {
|
|
19629
|
+
const end = getCurrentUnixTimestamp();
|
|
19630
|
+
metrics.duration = end - startTime;
|
|
19631
|
+
}
|
|
19632
|
+
if (response.usageMetadata) {
|
|
19633
|
+
const usage = response.usageMetadata;
|
|
19634
|
+
if (usage.promptTokenCount !== void 0) {
|
|
19635
|
+
metrics.prompt_tokens = usage.promptTokenCount;
|
|
19636
|
+
}
|
|
19637
|
+
if (usage.candidatesTokenCount !== void 0) {
|
|
19638
|
+
metrics.completion_tokens = usage.candidatesTokenCount;
|
|
19639
|
+
}
|
|
19640
|
+
if (usage.totalTokenCount !== void 0) {
|
|
19641
|
+
metrics.tokens = usage.totalTokenCount;
|
|
19642
|
+
}
|
|
19643
|
+
if (usage.cachedContentTokenCount !== void 0) {
|
|
19644
|
+
metrics.prompt_cached_tokens = usage.cachedContentTokenCount;
|
|
19645
|
+
}
|
|
19646
|
+
if (usage.thoughtsTokenCount !== void 0) {
|
|
19647
|
+
metrics.completion_reasoning_tokens = usage.thoughtsTokenCount;
|
|
19648
|
+
}
|
|
19649
|
+
}
|
|
19650
|
+
return metrics;
|
|
19651
|
+
}
|
|
19652
|
+
function aggregateGenerateContentChunks2(chunks, startTime) {
|
|
19653
|
+
const end = getCurrentUnixTimestamp();
|
|
19654
|
+
const metrics = {
|
|
19655
|
+
duration: end - startTime
|
|
19656
|
+
};
|
|
19657
|
+
let firstTokenTime = null;
|
|
19658
|
+
if (chunks.length > 0 && firstTokenTime === null) {
|
|
19659
|
+
firstTokenTime = getCurrentUnixTimestamp();
|
|
19660
|
+
metrics.time_to_first_token = firstTokenTime - startTime;
|
|
19661
|
+
}
|
|
19662
|
+
if (chunks.length === 0) {
|
|
19663
|
+
return { output: {}, metrics };
|
|
19664
|
+
}
|
|
19665
|
+
let text = "";
|
|
19666
|
+
let thoughtText = "";
|
|
19667
|
+
const otherParts = [];
|
|
19668
|
+
let usageMetadata = null;
|
|
19669
|
+
let lastResponse = null;
|
|
19670
|
+
for (const chunk of chunks) {
|
|
19671
|
+
lastResponse = chunk;
|
|
19672
|
+
if (chunk.usageMetadata) {
|
|
19673
|
+
usageMetadata = chunk.usageMetadata;
|
|
19674
|
+
}
|
|
19675
|
+
if (chunk.candidates && Array.isArray(chunk.candidates)) {
|
|
19676
|
+
for (const candidate of chunk.candidates) {
|
|
19677
|
+
if (candidate.content?.parts) {
|
|
19678
|
+
for (const part of candidate.content.parts) {
|
|
19679
|
+
if (part.text !== void 0) {
|
|
19680
|
+
if (part.thought) {
|
|
19681
|
+
thoughtText += part.text;
|
|
19682
|
+
} else {
|
|
19683
|
+
text += part.text;
|
|
19684
|
+
}
|
|
19685
|
+
} else if (part.functionCall) {
|
|
19686
|
+
otherParts.push({ functionCall: part.functionCall });
|
|
19687
|
+
} else if (part.codeExecutionResult) {
|
|
19688
|
+
otherParts.push({
|
|
19689
|
+
codeExecutionResult: part.codeExecutionResult
|
|
19690
|
+
});
|
|
19691
|
+
} else if (part.executableCode) {
|
|
19692
|
+
otherParts.push({ executableCode: part.executableCode });
|
|
19693
|
+
}
|
|
19694
|
+
}
|
|
19695
|
+
}
|
|
19696
|
+
}
|
|
19697
|
+
}
|
|
19698
|
+
}
|
|
19699
|
+
const output = {};
|
|
19700
|
+
const parts = [];
|
|
19701
|
+
if (thoughtText) {
|
|
19702
|
+
parts.push({ text: thoughtText, thought: true });
|
|
19703
|
+
}
|
|
19704
|
+
if (text) {
|
|
19705
|
+
parts.push({ text });
|
|
19706
|
+
}
|
|
19707
|
+
parts.push(...otherParts);
|
|
19708
|
+
if (parts.length > 0 && lastResponse?.candidates) {
|
|
19709
|
+
const candidates = [];
|
|
19710
|
+
for (const candidate of lastResponse.candidates) {
|
|
19711
|
+
const candidateDict = {
|
|
19712
|
+
content: {
|
|
19713
|
+
parts,
|
|
19714
|
+
role: "model"
|
|
19715
|
+
}
|
|
19716
|
+
};
|
|
19717
|
+
if (candidate.finishReason !== void 0) {
|
|
19718
|
+
candidateDict.finishReason = candidate.finishReason;
|
|
19719
|
+
}
|
|
19720
|
+
if (candidate.safetyRatings) {
|
|
19721
|
+
candidateDict.safetyRatings = candidate.safetyRatings;
|
|
19722
|
+
}
|
|
19723
|
+
candidates.push(candidateDict);
|
|
19724
|
+
}
|
|
19725
|
+
output.candidates = candidates;
|
|
19726
|
+
}
|
|
19727
|
+
if (usageMetadata) {
|
|
19728
|
+
output.usageMetadata = usageMetadata;
|
|
19729
|
+
if (usageMetadata.promptTokenCount !== void 0) {
|
|
19730
|
+
metrics.prompt_tokens = usageMetadata.promptTokenCount;
|
|
19731
|
+
}
|
|
19732
|
+
if (usageMetadata.candidatesTokenCount !== void 0) {
|
|
19733
|
+
metrics.completion_tokens = usageMetadata.candidatesTokenCount;
|
|
19734
|
+
}
|
|
19735
|
+
if (usageMetadata.totalTokenCount !== void 0) {
|
|
19736
|
+
metrics.tokens = usageMetadata.totalTokenCount;
|
|
19737
|
+
}
|
|
19738
|
+
if (usageMetadata.cachedContentTokenCount !== void 0) {
|
|
19739
|
+
metrics.prompt_cached_tokens = usageMetadata.cachedContentTokenCount;
|
|
19740
|
+
}
|
|
19741
|
+
if (usageMetadata.thoughtsTokenCount !== void 0) {
|
|
19742
|
+
metrics.completion_reasoning_tokens = usageMetadata.thoughtsTokenCount;
|
|
19743
|
+
}
|
|
19744
|
+
}
|
|
19745
|
+
if (text) {
|
|
19746
|
+
output.text = text;
|
|
19747
|
+
}
|
|
19748
|
+
return { output, metrics };
|
|
19749
|
+
}
|
|
19750
|
+
function tryToDict2(obj) {
|
|
19751
|
+
if (obj === null || obj === void 0) {
|
|
19752
|
+
return null;
|
|
19753
|
+
}
|
|
19754
|
+
if (typeof obj === "object") {
|
|
19755
|
+
if (typeof obj.toJSON === "function") {
|
|
19756
|
+
return obj.toJSON();
|
|
19757
|
+
}
|
|
19758
|
+
return obj;
|
|
19759
|
+
}
|
|
19760
|
+
return null;
|
|
19761
|
+
}
|
|
19762
|
+
|
|
19763
|
+
// src/instrumentation/braintrust-plugin.ts
|
|
19764
|
+
var BraintrustPlugin = class extends BasePlugin {
|
|
19765
|
+
config;
|
|
19766
|
+
openaiPlugin = null;
|
|
19767
|
+
anthropicPlugin = null;
|
|
19768
|
+
aiSDKPlugin = null;
|
|
19769
|
+
claudeAgentSDKPlugin = null;
|
|
19770
|
+
googleGenAIPlugin = null;
|
|
19771
|
+
constructor(config = {}) {
|
|
19772
|
+
super();
|
|
19773
|
+
this.config = config;
|
|
19774
|
+
}
|
|
19775
|
+
onEnable() {
|
|
19776
|
+
const integrations = this.config.integrations || {};
|
|
19777
|
+
if (integrations.openai !== false) {
|
|
19778
|
+
this.openaiPlugin = new OpenAIPlugin();
|
|
19779
|
+
this.openaiPlugin.enable();
|
|
19780
|
+
}
|
|
19781
|
+
if (integrations.anthropic !== false) {
|
|
19782
|
+
this.anthropicPlugin = new AnthropicPlugin();
|
|
19783
|
+
this.anthropicPlugin.enable();
|
|
19784
|
+
}
|
|
19785
|
+
if (integrations.aisdk !== false && integrations.vercel !== false) {
|
|
19786
|
+
this.aiSDKPlugin = new AISDKPlugin();
|
|
19787
|
+
this.aiSDKPlugin.enable();
|
|
19788
|
+
}
|
|
19789
|
+
if (integrations.claudeAgentSDK !== false) {
|
|
19790
|
+
this.claudeAgentSDKPlugin = new ClaudeAgentSDKPlugin();
|
|
19791
|
+
this.claudeAgentSDKPlugin.enable();
|
|
19792
|
+
}
|
|
19793
|
+
if (integrations.googleGenAI !== false && integrations.google !== false) {
|
|
19794
|
+
this.googleGenAIPlugin = new GoogleGenAIPlugin();
|
|
19795
|
+
this.googleGenAIPlugin.enable();
|
|
19796
|
+
}
|
|
19797
|
+
}
|
|
19798
|
+
onDisable() {
|
|
19799
|
+
if (this.openaiPlugin) {
|
|
19800
|
+
this.openaiPlugin.disable();
|
|
19801
|
+
this.openaiPlugin = null;
|
|
19802
|
+
}
|
|
19803
|
+
if (this.anthropicPlugin) {
|
|
19804
|
+
this.anthropicPlugin.disable();
|
|
19805
|
+
this.anthropicPlugin = null;
|
|
19806
|
+
}
|
|
19807
|
+
if (this.aiSDKPlugin) {
|
|
19808
|
+
this.aiSDKPlugin.disable();
|
|
19809
|
+
this.aiSDKPlugin = null;
|
|
19810
|
+
}
|
|
19811
|
+
if (this.claudeAgentSDKPlugin) {
|
|
19812
|
+
this.claudeAgentSDKPlugin.disable();
|
|
19813
|
+
this.claudeAgentSDKPlugin = null;
|
|
19814
|
+
}
|
|
19815
|
+
if (this.googleGenAIPlugin) {
|
|
19816
|
+
this.googleGenAIPlugin.disable();
|
|
19817
|
+
this.googleGenAIPlugin = null;
|
|
19818
|
+
}
|
|
19819
|
+
}
|
|
19820
|
+
};
|
|
19821
|
+
|
|
19822
|
+
// src/instrumentation/registry.ts
|
|
19823
|
+
var PluginRegistry = class {
|
|
19824
|
+
braintrustPlugin = null;
|
|
19825
|
+
config = {};
|
|
19826
|
+
enabled = false;
|
|
19827
|
+
/**
|
|
19828
|
+
* Configure which integrations should be enabled.
|
|
19829
|
+
* This must be called before any SDK imports to take effect.
|
|
19830
|
+
*/
|
|
19831
|
+
configure(config) {
|
|
19832
|
+
if (this.enabled) {
|
|
19833
|
+
console.warn(
|
|
19834
|
+
"Braintrust: Cannot configure instrumentation after it has been enabled. Call configureInstrumentation() before importing any AI SDKs."
|
|
19835
|
+
);
|
|
19836
|
+
return;
|
|
19837
|
+
}
|
|
19838
|
+
this.config = { ...this.config, ...config };
|
|
19839
|
+
}
|
|
19840
|
+
/**
|
|
19841
|
+
* Enable all configured plugins.
|
|
19842
|
+
* Called automatically when the library is loaded.
|
|
19843
|
+
*/
|
|
19844
|
+
enable() {
|
|
19845
|
+
if (this.enabled) {
|
|
19846
|
+
return;
|
|
19847
|
+
}
|
|
19848
|
+
this.enabled = true;
|
|
19849
|
+
const envConfig = this.readEnvConfig();
|
|
19850
|
+
const finalConfig = {
|
|
19851
|
+
integrations: {
|
|
19852
|
+
...this.getDefaultConfig(),
|
|
19853
|
+
...this.config.integrations,
|
|
19854
|
+
...envConfig.integrations
|
|
19855
|
+
}
|
|
19856
|
+
};
|
|
19857
|
+
this.braintrustPlugin = new BraintrustPlugin(finalConfig);
|
|
19858
|
+
this.braintrustPlugin.enable();
|
|
19859
|
+
}
|
|
19860
|
+
/**
|
|
19861
|
+
* Disable all plugins.
|
|
19862
|
+
* Primarily used for testing.
|
|
19863
|
+
*/
|
|
19864
|
+
disable() {
|
|
19865
|
+
if (!this.enabled) {
|
|
19866
|
+
return;
|
|
19867
|
+
}
|
|
19868
|
+
this.enabled = false;
|
|
19869
|
+
if (this.braintrustPlugin) {
|
|
19870
|
+
this.braintrustPlugin.disable();
|
|
19871
|
+
this.braintrustPlugin = null;
|
|
19872
|
+
}
|
|
19873
|
+
}
|
|
19874
|
+
/**
|
|
19875
|
+
* Check if instrumentation is enabled.
|
|
19876
|
+
*/
|
|
19877
|
+
isEnabled() {
|
|
19878
|
+
return this.enabled;
|
|
19879
|
+
}
|
|
19880
|
+
/**
|
|
19881
|
+
* Get default configuration (all integrations enabled).
|
|
19882
|
+
*/
|
|
19883
|
+
getDefaultConfig() {
|
|
19884
|
+
return {
|
|
19885
|
+
openai: true,
|
|
19886
|
+
anthropic: true,
|
|
19887
|
+
vercel: true,
|
|
19888
|
+
aisdk: true,
|
|
19889
|
+
google: true,
|
|
19890
|
+
claudeAgentSDK: true
|
|
19891
|
+
};
|
|
19892
|
+
}
|
|
19893
|
+
/**
|
|
19894
|
+
* Read configuration from environment variables.
|
|
19895
|
+
* Supports: BRAINTRUST_DISABLE_INSTRUMENTATION=openai,anthropic,...
|
|
19896
|
+
*/
|
|
19897
|
+
readEnvConfig() {
|
|
19898
|
+
const integrations = {};
|
|
19899
|
+
const disabledList = isomorph_default.getEnv("BRAINTRUST_DISABLE_INSTRUMENTATION");
|
|
19900
|
+
if (disabledList) {
|
|
19901
|
+
const disabled = disabledList.split(",").map((s) => s.trim().toLowerCase()).filter((s) => s.length > 0);
|
|
19902
|
+
for (const sdk of disabled) {
|
|
19903
|
+
integrations[sdk] = false;
|
|
19904
|
+
}
|
|
19905
|
+
}
|
|
19906
|
+
return { integrations };
|
|
19907
|
+
}
|
|
19908
|
+
};
|
|
19909
|
+
var registry = new PluginRegistry();
|
|
19910
|
+
function configureInstrumentation(config) {
|
|
19911
|
+
registry.configure(config);
|
|
19912
|
+
}
|
|
19913
|
+
|
|
16866
19914
|
// src/workerd/index.ts
|
|
16867
19915
|
configureWorkerd();
|