braintrust 3.1.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -0
- package/dev/dist/index.d.mts +187 -903
- package/dev/dist/index.d.ts +187 -903
- package/dev/dist/index.js +2942 -323
- package/dev/dist/index.mjs +2704 -85
- package/dist/auto-instrumentations/bundler/esbuild.cjs +401 -0
- package/dist/auto-instrumentations/bundler/esbuild.d.mts +8 -0
- package/dist/auto-instrumentations/bundler/esbuild.d.ts +8 -0
- package/dist/auto-instrumentations/bundler/esbuild.mjs +10 -0
- package/dist/auto-instrumentations/bundler/rollup.cjs +401 -0
- package/dist/auto-instrumentations/bundler/rollup.d.mts +8 -0
- package/dist/auto-instrumentations/bundler/rollup.d.ts +8 -0
- package/dist/auto-instrumentations/bundler/rollup.mjs +10 -0
- package/dist/auto-instrumentations/bundler/vite.cjs +401 -0
- package/dist/auto-instrumentations/bundler/vite.d.mts +8 -0
- package/dist/auto-instrumentations/bundler/vite.d.ts +8 -0
- package/dist/auto-instrumentations/bundler/vite.mjs +10 -0
- package/dist/auto-instrumentations/bundler/webpack.cjs +401 -0
- package/dist/auto-instrumentations/bundler/webpack.d.mts +8 -0
- package/dist/auto-instrumentations/bundler/webpack.d.ts +8 -0
- package/dist/auto-instrumentations/bundler/webpack.mjs +10 -0
- package/dist/auto-instrumentations/chunk-KVX7OFPD.mjs +288 -0
- package/dist/auto-instrumentations/chunk-OLOPGWTJ.mjs +89 -0
- package/dist/auto-instrumentations/chunk-XDBPUTVE.mjs +22 -0
- package/dist/auto-instrumentations/chunk-ZEC7BCL4.mjs +22 -0
- package/dist/auto-instrumentations/hook.mjs +378 -0
- package/dist/auto-instrumentations/index.cjs +318 -0
- package/dist/auto-instrumentations/index.d.mts +69 -0
- package/dist/auto-instrumentations/index.d.ts +69 -0
- package/dist/auto-instrumentations/index.mjs +14 -0
- package/dist/auto-instrumentations/loader/cjs-patch.cjs +116 -0
- package/dist/auto-instrumentations/loader/cjs-patch.d.mts +28 -0
- package/dist/auto-instrumentations/loader/cjs-patch.d.ts +28 -0
- package/dist/auto-instrumentations/loader/cjs-patch.mjs +66 -0
- package/dist/auto-instrumentations/loader/esm-hook.mjs +72 -0
- package/dist/auto-instrumentations/loader/get-package-version.cjs +46 -0
- package/dist/auto-instrumentations/loader/get-package-version.d.mts +7 -0
- package/dist/auto-instrumentations/loader/get-package-version.d.ts +7 -0
- package/dist/auto-instrumentations/loader/get-package-version.mjs +6 -0
- package/dist/auto-instrumentations/plugin-Df3qKIl2.d.mts +22 -0
- package/dist/auto-instrumentations/plugin-Df3qKIl2.d.ts +22 -0
- package/dist/browser.d.mts +484 -980
- package/dist/browser.d.ts +484 -980
- package/dist/browser.js +3437 -386
- package/dist/browser.mjs +3437 -386
- package/dist/cli.js +2890 -228
- package/dist/edge-light.d.mts +1 -1
- package/dist/edge-light.d.ts +1 -1
- package/dist/edge-light.js +3134 -86
- package/dist/edge-light.mjs +3134 -86
- package/dist/index.d.mts +484 -980
- package/dist/index.d.ts +484 -980
- package/dist/index.js +3754 -703
- package/dist/index.mjs +3615 -564
- package/dist/instrumentation/index.d.mts +323 -0
- package/dist/instrumentation/index.d.ts +323 -0
- package/dist/instrumentation/index.js +9942 -0
- package/dist/instrumentation/index.mjs +9902 -0
- package/dist/workerd.d.mts +1 -1
- package/dist/workerd.d.ts +1 -1
- package/dist/workerd.js +3134 -86
- package/dist/workerd.mjs +3134 -86
- package/package.json +46 -7
package/dist/workerd.mjs
CHANGED
|
@@ -1518,6 +1518,7 @@ var ApiKey = z6.object({
|
|
|
1518
1518
|
var TriggeredFunctionState = z6.object({
|
|
1519
1519
|
triggered_xact_id: z6.string(),
|
|
1520
1520
|
completed_xact_id: z6.union([z6.string(), z6.null()]).optional(),
|
|
1521
|
+
idempotency_key: z6.union([z6.string(), z6.null()]).optional(),
|
|
1521
1522
|
attempts: z6.number().int().gte(0).optional().default(0),
|
|
1522
1523
|
scope: z6.union([
|
|
1523
1524
|
z6.object({ type: z6.literal("span") }),
|
|
@@ -1550,7 +1551,8 @@ var AsyncScoringControl = z6.union([
|
|
|
1550
1551
|
scope: z6.union([
|
|
1551
1552
|
z6.object({ type: z6.literal("span") }),
|
|
1552
1553
|
z6.object({ type: z6.literal("trace") })
|
|
1553
|
-
])
|
|
1554
|
+
]),
|
|
1555
|
+
idempotency_key: z6.string().optional()
|
|
1554
1556
|
})
|
|
1555
1557
|
).min(1)
|
|
1556
1558
|
}),
|
|
@@ -1595,7 +1597,8 @@ var FunctionTypeEnum = z6.enum([
|
|
|
1595
1597
|
"facet",
|
|
1596
1598
|
"classifier",
|
|
1597
1599
|
"tag",
|
|
1598
|
-
"parameters"
|
|
1600
|
+
"parameters",
|
|
1601
|
+
"sandbox"
|
|
1599
1602
|
]);
|
|
1600
1603
|
var NullableSavedFunctionId = z6.union([
|
|
1601
1604
|
z6.object({
|
|
@@ -1610,66 +1613,14 @@ var NullableSavedFunctionId = z6.union([
|
|
|
1610
1613
|
}),
|
|
1611
1614
|
z6.null()
|
|
1612
1615
|
]);
|
|
1613
|
-
var TopicMapReport = z6.object({
|
|
1614
|
-
version: z6.literal(1),
|
|
1615
|
-
created_at: z6.string().optional(),
|
|
1616
|
-
settings: z6.object({
|
|
1617
|
-
algorithm: z6.enum(["hdbscan", "kmeans", "hierarchical"]),
|
|
1618
|
-
dimension_reduction: z6.enum(["umap", "pca", "none"]),
|
|
1619
|
-
vector_field: z6.string(),
|
|
1620
|
-
embedding_model: z6.string(),
|
|
1621
|
-
n_clusters: z6.union([z6.number(), z6.null()]).optional(),
|
|
1622
|
-
umap_dimensions: z6.union([z6.number(), z6.null()]).optional(),
|
|
1623
|
-
min_cluster_size: z6.union([z6.number(), z6.null()]).optional(),
|
|
1624
|
-
min_samples: z6.union([z6.number(), z6.null()]).optional()
|
|
1625
|
-
}),
|
|
1626
|
-
query_settings: z6.object({
|
|
1627
|
-
hierarchy_threshold: z6.union([z6.number(), z6.null()]),
|
|
1628
|
-
auto_naming: z6.boolean(),
|
|
1629
|
-
skip_cache: z6.boolean(),
|
|
1630
|
-
viz_mode: z6.enum(["bar", "scatter"]),
|
|
1631
|
-
naming_model: z6.string()
|
|
1632
|
-
}).partial(),
|
|
1633
|
-
clusters: z6.array(
|
|
1634
|
-
z6.object({
|
|
1635
|
-
cluster_id: z6.number(),
|
|
1636
|
-
parent_cluster_id: z6.union([z6.number(), z6.null()]).optional(),
|
|
1637
|
-
topic_id: z6.string(),
|
|
1638
|
-
count: z6.number(),
|
|
1639
|
-
sample_texts: z6.array(z6.string()),
|
|
1640
|
-
samples: z6.array(
|
|
1641
|
-
z6.object({
|
|
1642
|
-
id: z6.string(),
|
|
1643
|
-
text: z6.string(),
|
|
1644
|
-
root_span_id: z6.string(),
|
|
1645
|
-
span_id: z6.string()
|
|
1646
|
-
})
|
|
1647
|
-
),
|
|
1648
|
-
name: z6.string().optional(),
|
|
1649
|
-
description: z6.string().optional(),
|
|
1650
|
-
keywords: z6.array(z6.string()).optional(),
|
|
1651
|
-
centroid: z6.array(z6.number()).optional(),
|
|
1652
|
-
parent_id: z6.union([z6.number(), z6.null()]).optional(),
|
|
1653
|
-
is_leaf: z6.boolean().optional(),
|
|
1654
|
-
depth: z6.number().optional()
|
|
1655
|
-
})
|
|
1656
|
-
),
|
|
1657
|
-
embedding_points: z6.array(
|
|
1658
|
-
z6.object({
|
|
1659
|
-
x: z6.number(),
|
|
1660
|
-
y: z6.number(),
|
|
1661
|
-
cluster: z6.number(),
|
|
1662
|
-
text: z6.string().optional()
|
|
1663
|
-
})
|
|
1664
|
-
).optional()
|
|
1665
|
-
});
|
|
1666
1616
|
var TopicMapData = z6.object({
|
|
1667
1617
|
type: z6.literal("topic_map"),
|
|
1668
1618
|
source_facet: z6.string(),
|
|
1669
1619
|
embedding_model: z6.string(),
|
|
1670
|
-
bundle_key: z6.string(),
|
|
1671
|
-
|
|
1672
|
-
|
|
1620
|
+
bundle_key: z6.string().optional(),
|
|
1621
|
+
report_key: z6.string().optional(),
|
|
1622
|
+
topic_names: z6.record(z6.string()).optional(),
|
|
1623
|
+
distance_threshold: z6.number().optional()
|
|
1673
1624
|
});
|
|
1674
1625
|
var BatchedFacetData = z6.object({
|
|
1675
1626
|
type: z6.literal("batched_facet"),
|
|
@@ -1684,11 +1635,13 @@ var BatchedFacetData = z6.object({
|
|
|
1684
1635
|
})
|
|
1685
1636
|
),
|
|
1686
1637
|
topic_maps: z6.record(
|
|
1687
|
-
z6.
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1638
|
+
z6.array(
|
|
1639
|
+
z6.object({
|
|
1640
|
+
function_name: z6.string(),
|
|
1641
|
+
topic_map_id: z6.string().optional(),
|
|
1642
|
+
topic_map_data: TopicMapData
|
|
1643
|
+
})
|
|
1644
|
+
)
|
|
1692
1645
|
).optional()
|
|
1693
1646
|
});
|
|
1694
1647
|
var BraintrustModelParams = z6.object({
|
|
@@ -1867,9 +1820,20 @@ var CodeBundle = z6.object({
|
|
|
1867
1820
|
z6.object({ type: z6.literal("scorer"), index: z6.number().int().gte(0) })
|
|
1868
1821
|
])
|
|
1869
1822
|
}),
|
|
1870
|
-
z6.object({ type: z6.literal("function"), index: z6.number().int().gte(0) })
|
|
1823
|
+
z6.object({ type: z6.literal("function"), index: z6.number().int().gte(0) }),
|
|
1824
|
+
z6.object({
|
|
1825
|
+
type: z6.literal("sandbox"),
|
|
1826
|
+
sandbox_spec: z6.union([
|
|
1827
|
+
z6.object({ provider: z6.literal("modal"), snapshot_ref: z6.string() }),
|
|
1828
|
+
z6.object({ provider: z6.literal("lambda") })
|
|
1829
|
+
]),
|
|
1830
|
+
entrypoints: z6.array(z6.string()).optional(),
|
|
1831
|
+
eval_name: z6.string(),
|
|
1832
|
+
parameters: z6.object({}).partial().passthrough().optional(),
|
|
1833
|
+
evaluator_definition: z6.unknown().optional()
|
|
1834
|
+
})
|
|
1871
1835
|
]),
|
|
1872
|
-
bundle_id: z6.string(),
|
|
1836
|
+
bundle_id: z6.union([z6.string(), z6.null()]).optional(),
|
|
1873
1837
|
preview: z6.union([z6.string(), z6.null()]).optional()
|
|
1874
1838
|
});
|
|
1875
1839
|
var Dataset = z6.object({
|
|
@@ -1971,7 +1935,7 @@ var EnvVar = z6.object({
|
|
|
1971
1935
|
used: z6.union([z6.string(), z6.null()]).optional(),
|
|
1972
1936
|
metadata: z6.union([z6.object({}).partial().passthrough(), z6.null()]).optional(),
|
|
1973
1937
|
secret_type: z6.union([z6.string(), z6.null()]).optional(),
|
|
1974
|
-
secret_category: z6.enum(["env_var", "ai_provider"]).optional().default("env_var")
|
|
1938
|
+
secret_category: z6.enum(["env_var", "ai_provider", "sandbox_provider"]).optional().default("env_var")
|
|
1975
1939
|
});
|
|
1976
1940
|
var EvalStatusPageTheme = z6.enum(["light", "dark"]);
|
|
1977
1941
|
var EvalStatusPageConfig = z6.object({
|
|
@@ -2273,7 +2237,8 @@ var FunctionTypeEnumNullish = z6.union([
|
|
|
2273
2237
|
"facet",
|
|
2274
2238
|
"classifier",
|
|
2275
2239
|
"tag",
|
|
2276
|
-
"parameters"
|
|
2240
|
+
"parameters",
|
|
2241
|
+
"sandbox"
|
|
2277
2242
|
]),
|
|
2278
2243
|
z6.null()
|
|
2279
2244
|
]);
|
|
@@ -2506,7 +2471,8 @@ var FunctionObjectType = z6.enum([
|
|
|
2506
2471
|
"preprocessor",
|
|
2507
2472
|
"facet",
|
|
2508
2473
|
"classifier",
|
|
2509
|
-
"parameters"
|
|
2474
|
+
"parameters",
|
|
2475
|
+
"sandbox"
|
|
2510
2476
|
]);
|
|
2511
2477
|
var FunctionOutputType = z6.enum([
|
|
2512
2478
|
"completion",
|
|
@@ -2642,6 +2608,7 @@ var Organization = z6.object({
|
|
|
2642
2608
|
name: z6.string(),
|
|
2643
2609
|
api_url: z6.union([z6.string(), z6.null()]).optional(),
|
|
2644
2610
|
is_universal_api: z6.union([z6.boolean(), z6.null()]).optional(),
|
|
2611
|
+
is_dataplane_private: z6.union([z6.boolean(), z6.null()]).optional(),
|
|
2645
2612
|
proxy_url: z6.union([z6.string(), z6.null()]).optional(),
|
|
2646
2613
|
realtime_url: z6.union([z6.string(), z6.null()]).optional(),
|
|
2647
2614
|
created: z6.union([z6.string(), z6.null()]).optional(),
|
|
@@ -2666,7 +2633,7 @@ var ProjectSettings = z6.union([
|
|
|
2666
2633
|
z6.array(
|
|
2667
2634
|
z6.object({
|
|
2668
2635
|
url: z6.string(),
|
|
2669
|
-
name: z6.string(),
|
|
2636
|
+
name: z6.union([z6.string(), z6.null()]).optional(),
|
|
2670
2637
|
description: z6.union([z6.string(), z6.null()]).optional()
|
|
2671
2638
|
})
|
|
2672
2639
|
),
|
|
@@ -2692,6 +2659,25 @@ var RetentionObjectType = z6.enum([
|
|
|
2692
2659
|
"experiment",
|
|
2693
2660
|
"dataset"
|
|
2694
2661
|
]);
|
|
2662
|
+
var TopicMapFunctionAutomation = z6.object({
|
|
2663
|
+
function: SavedFunctionId.and(z6.unknown()),
|
|
2664
|
+
btql_filter: z6.union([z6.string(), z6.null()]).optional()
|
|
2665
|
+
});
|
|
2666
|
+
var TopicAutomationDataScope = z6.union([
|
|
2667
|
+
z6.object({ type: z6.literal("project_logs") }),
|
|
2668
|
+
z6.object({ type: z6.literal("project_experiments") }),
|
|
2669
|
+
z6.object({ type: z6.literal("experiment"), experiment_id: z6.string() }),
|
|
2670
|
+
z6.null()
|
|
2671
|
+
]);
|
|
2672
|
+
var TopicAutomationConfig = z6.object({
|
|
2673
|
+
event_type: z6.literal("topic"),
|
|
2674
|
+
sampling_rate: z6.number().gte(0).lte(1),
|
|
2675
|
+
facet_functions: z6.array(SavedFunctionId),
|
|
2676
|
+
topic_map_functions: z6.array(TopicMapFunctionAutomation),
|
|
2677
|
+
scope: z6.union([SpanScope, TraceScope, GroupScope, z6.null()]).optional(),
|
|
2678
|
+
data_scope: TopicAutomationDataScope.optional(),
|
|
2679
|
+
btql_filter: z6.union([z6.string(), z6.null()]).optional()
|
|
2680
|
+
});
|
|
2695
2681
|
var ProjectAutomation = z6.object({
|
|
2696
2682
|
id: z6.string().uuid(),
|
|
2697
2683
|
project_id: z6.string().uuid(),
|
|
@@ -2748,7 +2734,8 @@ var ProjectAutomation = z6.object({
|
|
|
2748
2734
|
message_template: z6.string().optional()
|
|
2749
2735
|
})
|
|
2750
2736
|
])
|
|
2751
|
-
})
|
|
2737
|
+
}),
|
|
2738
|
+
TopicAutomationConfig
|
|
2752
2739
|
])
|
|
2753
2740
|
});
|
|
2754
2741
|
var ProjectLogsEvent = z6.object({
|
|
@@ -2938,6 +2925,8 @@ var RunEval = z6.object({
|
|
|
2938
2925
|
}),
|
|
2939
2926
|
z6.object({ data: z6.array(z6.unknown()) })
|
|
2940
2927
|
]),
|
|
2928
|
+
name: z6.string().optional(),
|
|
2929
|
+
parameters: z6.object({}).partial().passthrough().optional(),
|
|
2941
2930
|
task: FunctionId.and(z6.unknown()),
|
|
2942
2931
|
scores: z6.array(FunctionId),
|
|
2943
2932
|
experiment_name: z6.string().optional(),
|
|
@@ -4645,7 +4634,7 @@ var HTTPConnection = class _HTTPConnection {
|
|
|
4645
4634
|
try {
|
|
4646
4635
|
const resp = await this.get("ping");
|
|
4647
4636
|
return resp.status === 200;
|
|
4648
|
-
} catch
|
|
4637
|
+
} catch {
|
|
4649
4638
|
return false;
|
|
4650
4639
|
}
|
|
4651
4640
|
}
|
|
@@ -5154,9 +5143,9 @@ function logFeedbackImpl(state, parentObjectType, parentObjectId, {
|
|
|
5154
5143
|
expected,
|
|
5155
5144
|
tags
|
|
5156
5145
|
});
|
|
5157
|
-
|
|
5158
|
-
updateEvent = Object.fromEntries(
|
|
5159
|
-
Object.entries(
|
|
5146
|
+
const { metadata, ...rawUpdateEvent } = deepCopyEvent(validatedEvent);
|
|
5147
|
+
const updateEvent = Object.fromEntries(
|
|
5148
|
+
Object.entries(rawUpdateEvent).filter(([_, v]) => !isEmpty2(v))
|
|
5160
5149
|
);
|
|
5161
5150
|
const parentIds = async () => new SpanComponentsV3({
|
|
5162
5151
|
object_type: parentObjectType,
|
|
@@ -7344,7 +7333,8 @@ function validateAndSanitizeExperimentLogPartialArgs(event) {
|
|
|
7344
7333
|
if (Array.isArray(event.scores)) {
|
|
7345
7334
|
throw new Error("scores must be an object, not an array");
|
|
7346
7335
|
}
|
|
7347
|
-
for (
|
|
7336
|
+
for (const [name, rawScore] of Object.entries(event.scores)) {
|
|
7337
|
+
let score = rawScore;
|
|
7348
7338
|
if (typeof name !== "string") {
|
|
7349
7339
|
throw new Error("score names must be strings");
|
|
7350
7340
|
}
|
|
@@ -7786,7 +7776,11 @@ var Experiment2 = class extends ObjectFetcher {
|
|
|
7786
7776
|
* @returns A summary of the experiment, including the scores (compared to the closest reference experiment) and metadata.
|
|
7787
7777
|
*/
|
|
7788
7778
|
async summarize(options = {}) {
|
|
7789
|
-
|
|
7779
|
+
const {
|
|
7780
|
+
summarizeScores = true,
|
|
7781
|
+
comparisonExperimentId: comparisonExperimentIdOpt
|
|
7782
|
+
} = options || {};
|
|
7783
|
+
let comparisonExperimentId = comparisonExperimentIdOpt;
|
|
7790
7784
|
const state = await this.getState();
|
|
7791
7785
|
const projectUrl = `${state.appPublicUrl}/app/${encodeURIComponent(
|
|
7792
7786
|
state.orgName
|
|
@@ -8970,8 +8964,7 @@ var RemoteEvalParameters = class {
|
|
|
8970
8964
|
return true;
|
|
8971
8965
|
}
|
|
8972
8966
|
static isParameters(x) {
|
|
8973
|
-
return typeof x === "object" && x !== null && "__braintrust_parameters_marker" in x &&
|
|
8974
|
-
x.__braintrust_parameters_marker === true;
|
|
8967
|
+
return typeof x === "object" && x !== null && "__braintrust_parameters_marker" in x && x.__braintrust_parameters_marker === true;
|
|
8975
8968
|
}
|
|
8976
8969
|
};
|
|
8977
8970
|
var TEST_API_KEY = "___TEST_API_KEY__THIS_IS_NOT_REAL___";
|
|
@@ -9154,6 +9147,7 @@ __export(exports_exports, {
|
|
|
9154
9147
|
addAzureBlobHeaders: () => addAzureBlobHeaders,
|
|
9155
9148
|
braintrustStreamChunkSchema: () => braintrustStreamChunkSchema,
|
|
9156
9149
|
buildLocalSummary: () => buildLocalSummary,
|
|
9150
|
+
configureInstrumentation: () => configureInstrumentation,
|
|
9157
9151
|
constructLogs3OverflowRequest: () => constructLogs3OverflowRequest,
|
|
9158
9152
|
createFinalValuePassThroughStream: () => createFinalValuePassThroughStream,
|
|
9159
9153
|
currentExperiment: () => currentExperiment,
|
|
@@ -9229,7 +9223,8 @@ __export(exports_exports, {
|
|
|
9229
9223
|
wrapMastraAgent: () => wrapMastraAgent,
|
|
9230
9224
|
wrapOpenAI: () => wrapOpenAI,
|
|
9231
9225
|
wrapOpenAIv4: () => wrapOpenAIv4,
|
|
9232
|
-
wrapTraced: () => wrapTraced
|
|
9226
|
+
wrapTraced: () => wrapTraced,
|
|
9227
|
+
wrapVitest: () => wrapVitest
|
|
9233
9228
|
});
|
|
9234
9229
|
|
|
9235
9230
|
// src/functions/invoke.ts
|
|
@@ -9385,7 +9380,7 @@ function parseEventFromResponseCreateResult(result) {
|
|
|
9385
9380
|
data.output = processImagesInOutput(result.output);
|
|
9386
9381
|
}
|
|
9387
9382
|
if (result) {
|
|
9388
|
-
const { output, usage, ...metadata } = result;
|
|
9383
|
+
const { output: _output, usage: _usage, ...metadata } = result;
|
|
9389
9384
|
if (Object.keys(metadata).length > 0) {
|
|
9390
9385
|
data.metadata = metadata;
|
|
9391
9386
|
}
|
|
@@ -9448,7 +9443,7 @@ function parseEventFromResponseParseResult(result) {
|
|
|
9448
9443
|
data.output = processImagesInOutput(result.output);
|
|
9449
9444
|
}
|
|
9450
9445
|
if (result) {
|
|
9451
|
-
const { output, usage, ...metadata } = result;
|
|
9446
|
+
const { output: _output, usage: _usage, ...metadata } = result;
|
|
9452
9447
|
if (Object.keys(metadata).length > 0) {
|
|
9453
9448
|
data.metadata = metadata;
|
|
9454
9449
|
}
|
|
@@ -9492,7 +9487,7 @@ function parseLogFromItem(item) {
|
|
|
9492
9487
|
data.output = processImagesInOutput(response.output);
|
|
9493
9488
|
}
|
|
9494
9489
|
if (response) {
|
|
9495
|
-
const { usage, output, ...metadata } = response;
|
|
9490
|
+
const { usage: _usage, output: _output, ...metadata } = response;
|
|
9496
9491
|
if (Object.keys(metadata).length > 0) {
|
|
9497
9492
|
data.metadata = metadata;
|
|
9498
9493
|
}
|
|
@@ -9883,7 +9878,7 @@ function wrapChatCompletion(completion) {
|
|
|
9883
9878
|
);
|
|
9884
9879
|
const { data: ret, response } = await completionResponse.withResponse();
|
|
9885
9880
|
logHeaders(response, span);
|
|
9886
|
-
const { messages, ...rest } = params;
|
|
9881
|
+
const { messages: _messages, ...rest } = params;
|
|
9887
9882
|
span.log({
|
|
9888
9883
|
metadata: {
|
|
9889
9884
|
...rest
|
|
@@ -10206,7 +10201,7 @@ function convertDataToBlob(data, mediaType) {
|
|
|
10206
10201
|
} else if (typeof Buffer !== "undefined" && data instanceof Buffer) {
|
|
10207
10202
|
return new Blob([data], { type: mediaType });
|
|
10208
10203
|
}
|
|
10209
|
-
} catch
|
|
10204
|
+
} catch {
|
|
10210
10205
|
return null;
|
|
10211
10206
|
}
|
|
10212
10207
|
return null;
|
|
@@ -12479,7 +12474,6 @@ function apiPromiseProxy2(apiPromise, span, onThen) {
|
|
|
12479
12474
|
return function(onFulfilled, onRejected) {
|
|
12480
12475
|
return thenFunc.call(
|
|
12481
12476
|
target,
|
|
12482
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
12483
12477
|
async (result) => {
|
|
12484
12478
|
try {
|
|
12485
12479
|
const processed = onThen(result);
|
|
@@ -13691,6 +13685,582 @@ function tryToDict(obj) {
|
|
|
13691
13685
|
return null;
|
|
13692
13686
|
}
|
|
13693
13687
|
|
|
13688
|
+
// src/wrappers/vitest/context-manager.ts
|
|
13689
|
+
var VitestContextManager = class {
|
|
13690
|
+
/**
|
|
13691
|
+
* AsyncLocalStorage for experiment context isolation.
|
|
13692
|
+
* Each async execution flow (test, concurrent test, worker thread) gets its own context.
|
|
13693
|
+
*/
|
|
13694
|
+
contextStorage;
|
|
13695
|
+
constructor() {
|
|
13696
|
+
this.contextStorage = isomorph_default.newAsyncLocalStorage();
|
|
13697
|
+
}
|
|
13698
|
+
getCurrentContext() {
|
|
13699
|
+
return this.contextStorage.getStore();
|
|
13700
|
+
}
|
|
13701
|
+
setContext(context) {
|
|
13702
|
+
this.contextStorage.enterWith(context);
|
|
13703
|
+
}
|
|
13704
|
+
runInContext(context, callback) {
|
|
13705
|
+
return this.contextStorage.run(context, callback);
|
|
13706
|
+
}
|
|
13707
|
+
createChildContext(dataset, experiment) {
|
|
13708
|
+
const parent = this.getCurrentContext();
|
|
13709
|
+
return {
|
|
13710
|
+
dataset,
|
|
13711
|
+
experiment,
|
|
13712
|
+
datasetExamples: /* @__PURE__ */ new Map(),
|
|
13713
|
+
parent,
|
|
13714
|
+
flushResolved: true,
|
|
13715
|
+
passed: 0,
|
|
13716
|
+
failed: 0
|
|
13717
|
+
};
|
|
13718
|
+
}
|
|
13719
|
+
};
|
|
13720
|
+
var _contextManager;
|
|
13721
|
+
function getVitestContextManager() {
|
|
13722
|
+
if (!_contextManager) {
|
|
13723
|
+
_contextManager = new VitestContextManager();
|
|
13724
|
+
}
|
|
13725
|
+
return _contextManager;
|
|
13726
|
+
}
|
|
13727
|
+
|
|
13728
|
+
// src/wrappers/vitest/flush-manager.ts
|
|
13729
|
+
var FlushCoordinator = class {
|
|
13730
|
+
activeFlushes = /* @__PURE__ */ new Map();
|
|
13731
|
+
async coordinateFlush(context, config) {
|
|
13732
|
+
if (!context) return;
|
|
13733
|
+
const experimentId = await context.experiment.id;
|
|
13734
|
+
if (this.activeFlushes.has(experimentId)) {
|
|
13735
|
+
return this.activeFlushes.get(experimentId);
|
|
13736
|
+
}
|
|
13737
|
+
const flushPromise = this.doFlush(context, config);
|
|
13738
|
+
this.activeFlushes.set(experimentId, flushPromise);
|
|
13739
|
+
try {
|
|
13740
|
+
await flushPromise;
|
|
13741
|
+
} finally {
|
|
13742
|
+
this.activeFlushes.delete(experimentId);
|
|
13743
|
+
}
|
|
13744
|
+
}
|
|
13745
|
+
async doFlush(context, config) {
|
|
13746
|
+
let summary;
|
|
13747
|
+
try {
|
|
13748
|
+
summary = await context.experiment.summarize();
|
|
13749
|
+
} catch (error) {
|
|
13750
|
+
console.warn("Failed to generate experiment summary:", error);
|
|
13751
|
+
}
|
|
13752
|
+
try {
|
|
13753
|
+
await context.experiment.flush();
|
|
13754
|
+
} catch (error) {
|
|
13755
|
+
console.warn("Failed to flush experiment:", error);
|
|
13756
|
+
throw error;
|
|
13757
|
+
}
|
|
13758
|
+
if (summary && (config.displaySummary ?? true)) {
|
|
13759
|
+
console.log(formatExperimentSummary(summary));
|
|
13760
|
+
}
|
|
13761
|
+
}
|
|
13762
|
+
};
|
|
13763
|
+
var flushCoordinator = new FlushCoordinator();
|
|
13764
|
+
async function flushExperimentWithSync(context, config) {
|
|
13765
|
+
return flushCoordinator.coordinateFlush(context, config);
|
|
13766
|
+
}
|
|
13767
|
+
|
|
13768
|
+
// src/wrappers/vitest/scorers.ts
|
|
13769
|
+
async function runScorers(args) {
|
|
13770
|
+
const { scorers, output, expected, input, metadata, span } = args;
|
|
13771
|
+
const scorerArgs = {
|
|
13772
|
+
output,
|
|
13773
|
+
expected,
|
|
13774
|
+
input,
|
|
13775
|
+
metadata: metadata || {}
|
|
13776
|
+
};
|
|
13777
|
+
for (const scorer of scorers) {
|
|
13778
|
+
try {
|
|
13779
|
+
const result = await scorer(scorerArgs);
|
|
13780
|
+
const scores = normalizeScores(result);
|
|
13781
|
+
for (const score of scores) {
|
|
13782
|
+
if (score.metadata && Object.keys(score.metadata).length > 0) {
|
|
13783
|
+
span.log({
|
|
13784
|
+
scores: { [score.name]: score.score },
|
|
13785
|
+
metadata: score.metadata
|
|
13786
|
+
});
|
|
13787
|
+
} else {
|
|
13788
|
+
span.log({
|
|
13789
|
+
scores: { [score.name]: score.score }
|
|
13790
|
+
});
|
|
13791
|
+
}
|
|
13792
|
+
}
|
|
13793
|
+
} catch (scorerError) {
|
|
13794
|
+
console.warn("Braintrust: Scorer failed:", scorerError);
|
|
13795
|
+
span.log({
|
|
13796
|
+
metadata: {
|
|
13797
|
+
scorer_error: scorerError instanceof Error ? { message: scorerError.message, name: scorerError.name } : String(scorerError)
|
|
13798
|
+
}
|
|
13799
|
+
});
|
|
13800
|
+
}
|
|
13801
|
+
}
|
|
13802
|
+
}
|
|
13803
|
+
function isScore(val) {
|
|
13804
|
+
return "name" in val && "score" in val;
|
|
13805
|
+
}
|
|
13806
|
+
function normalizeScores(result) {
|
|
13807
|
+
if (result === null || result === void 0) {
|
|
13808
|
+
return [];
|
|
13809
|
+
}
|
|
13810
|
+
if (typeof result === "number") {
|
|
13811
|
+
return [{ name: "score", score: result }];
|
|
13812
|
+
}
|
|
13813
|
+
if (Array.isArray(result)) {
|
|
13814
|
+
return result.filter((s) => s !== null && s !== void 0);
|
|
13815
|
+
}
|
|
13816
|
+
if (typeof result === "object" && result !== null && isScore(result)) {
|
|
13817
|
+
return [result];
|
|
13818
|
+
}
|
|
13819
|
+
return [];
|
|
13820
|
+
}
|
|
13821
|
+
|
|
13822
|
+
// src/wrappers/vitest/wrapper.ts
|
|
13823
|
+
function formatExperimentSummary(summary) {
|
|
13824
|
+
const lines = [];
|
|
13825
|
+
lines.push("\n\u250C\u2500 Braintrust Experiment Summary \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510");
|
|
13826
|
+
lines.push(`\u2502 Experiment: ${summary.experimentName}`);
|
|
13827
|
+
if (Object.keys(summary.scores).length > 0) {
|
|
13828
|
+
lines.push("\u2502");
|
|
13829
|
+
lines.push("\u2502 Scores:");
|
|
13830
|
+
for (const [name, score] of Object.entries(summary.scores)) {
|
|
13831
|
+
const percent = (score.score * 100).toFixed(2);
|
|
13832
|
+
lines.push(`\u2502 ${name}: ${percent}%`);
|
|
13833
|
+
}
|
|
13834
|
+
}
|
|
13835
|
+
if (summary.metrics && Object.keys(summary.metrics).length > 0) {
|
|
13836
|
+
lines.push("\u2502");
|
|
13837
|
+
lines.push("\u2502 Metrics:");
|
|
13838
|
+
for (const [name, metric] of Object.entries(summary.metrics)) {
|
|
13839
|
+
const value = Number.isInteger(metric.metric) ? metric.metric.toFixed(0) : metric.metric.toFixed(2);
|
|
13840
|
+
const formatted = metric.unit === "$" ? `${metric.unit}${value}` : `${value}${metric.unit}`;
|
|
13841
|
+
lines.push(`\u2502 ${name}: ${formatted}`);
|
|
13842
|
+
}
|
|
13843
|
+
}
|
|
13844
|
+
if (summary.experimentUrl) {
|
|
13845
|
+
lines.push("\u2502");
|
|
13846
|
+
lines.push(`\u2502 View results: ${summary.experimentUrl}`);
|
|
13847
|
+
}
|
|
13848
|
+
lines.push("\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n");
|
|
13849
|
+
return lines.join("\n");
|
|
13850
|
+
}
|
|
13851
|
+
function getExperimentContext() {
|
|
13852
|
+
return getVitestContextManager().getCurrentContext() ?? null;
|
|
13853
|
+
}
|
|
13854
|
+
function wrapTest(originalTest, config) {
|
|
13855
|
+
const wrapBare = (testFn) => {
|
|
13856
|
+
const wrapped = function(name, configOrFn, maybeFn) {
|
|
13857
|
+
const isEnhanced = typeof configOrFn !== "function";
|
|
13858
|
+
const testConfig = isEnhanced ? configOrFn : void 0;
|
|
13859
|
+
if (isEnhanced && testConfig?.data && Array.isArray(testConfig.data)) {
|
|
13860
|
+
const dataRecords = testConfig.data;
|
|
13861
|
+
const testFn2 = maybeFn;
|
|
13862
|
+
if (!testFn2) {
|
|
13863
|
+
throw new Error(
|
|
13864
|
+
"Braintrust: test function required when using data array"
|
|
13865
|
+
);
|
|
13866
|
+
}
|
|
13867
|
+
dataRecords.forEach((record, index) => {
|
|
13868
|
+
const mergedConfig = {
|
|
13869
|
+
...testConfig,
|
|
13870
|
+
input: record.input,
|
|
13871
|
+
expected: record.expected,
|
|
13872
|
+
metadata: { ...testConfig.metadata, ...record.metadata },
|
|
13873
|
+
tags: [
|
|
13874
|
+
...testConfig.tags || [],
|
|
13875
|
+
...record.tags || []
|
|
13876
|
+
],
|
|
13877
|
+
data: void 0
|
|
13878
|
+
};
|
|
13879
|
+
wrappedTest(`${name} [${index}]`, mergedConfig, testFn2);
|
|
13880
|
+
});
|
|
13881
|
+
return;
|
|
13882
|
+
}
|
|
13883
|
+
let vitestOptions;
|
|
13884
|
+
if (testConfig) {
|
|
13885
|
+
const {
|
|
13886
|
+
input: _input,
|
|
13887
|
+
expected: _expected,
|
|
13888
|
+
metadata: _metadata,
|
|
13889
|
+
tags: _tags,
|
|
13890
|
+
scorers: _scorers,
|
|
13891
|
+
data: _data,
|
|
13892
|
+
...rest
|
|
13893
|
+
} = testConfig;
|
|
13894
|
+
vitestOptions = rest;
|
|
13895
|
+
}
|
|
13896
|
+
const hasVitestOptions = vitestOptions && Object.keys(vitestOptions).length > 0;
|
|
13897
|
+
const registrationContext = getExperimentContext();
|
|
13898
|
+
const testImplementation = async (vitestContext) => {
|
|
13899
|
+
const experimentContext = getExperimentContext() ?? registrationContext;
|
|
13900
|
+
const experiment = experimentContext?.experiment;
|
|
13901
|
+
if (config.onProgress) {
|
|
13902
|
+
config.onProgress({ type: "test_start", testName: name });
|
|
13903
|
+
}
|
|
13904
|
+
const startTime = performance.now();
|
|
13905
|
+
let passed = false;
|
|
13906
|
+
try {
|
|
13907
|
+
if (!experiment) {
|
|
13908
|
+
if (testConfig && maybeFn) {
|
|
13909
|
+
const params = {
|
|
13910
|
+
input: testConfig.input,
|
|
13911
|
+
expected: testConfig.expected,
|
|
13912
|
+
metadata: testConfig.metadata
|
|
13913
|
+
};
|
|
13914
|
+
const context = {
|
|
13915
|
+
...vitestContext,
|
|
13916
|
+
...params
|
|
13917
|
+
};
|
|
13918
|
+
const result2 = await maybeFn(context);
|
|
13919
|
+
passed = true;
|
|
13920
|
+
return result2;
|
|
13921
|
+
} else if (typeof configOrFn === "function") {
|
|
13922
|
+
const result2 = await configOrFn(vitestContext);
|
|
13923
|
+
passed = true;
|
|
13924
|
+
return result2;
|
|
13925
|
+
}
|
|
13926
|
+
passed = true;
|
|
13927
|
+
return;
|
|
13928
|
+
}
|
|
13929
|
+
const result = await experiment.traced(
|
|
13930
|
+
async (span) => {
|
|
13931
|
+
let testResult;
|
|
13932
|
+
try {
|
|
13933
|
+
if (testConfig && maybeFn) {
|
|
13934
|
+
const params = {
|
|
13935
|
+
input: testConfig.input,
|
|
13936
|
+
expected: testConfig.expected,
|
|
13937
|
+
metadata: testConfig.metadata
|
|
13938
|
+
};
|
|
13939
|
+
const context = {
|
|
13940
|
+
...vitestContext,
|
|
13941
|
+
...params
|
|
13942
|
+
};
|
|
13943
|
+
testResult = await maybeFn(context);
|
|
13944
|
+
} else if (typeof configOrFn === "function") {
|
|
13945
|
+
testResult = await configOrFn(vitestContext);
|
|
13946
|
+
}
|
|
13947
|
+
if (testConfig?.scorers && testConfig.scorers.length > 0) {
|
|
13948
|
+
await runScorers({
|
|
13949
|
+
scorers: testConfig.scorers,
|
|
13950
|
+
output: testResult,
|
|
13951
|
+
expected: testConfig.expected,
|
|
13952
|
+
input: testConfig.input,
|
|
13953
|
+
metadata: testConfig.metadata,
|
|
13954
|
+
span
|
|
13955
|
+
});
|
|
13956
|
+
}
|
|
13957
|
+
span.log({
|
|
13958
|
+
scores: {
|
|
13959
|
+
pass: 1
|
|
13960
|
+
}
|
|
13961
|
+
});
|
|
13962
|
+
if (testResult !== void 0) {
|
|
13963
|
+
span.log({
|
|
13964
|
+
output: testResult
|
|
13965
|
+
});
|
|
13966
|
+
}
|
|
13967
|
+
} catch (error) {
|
|
13968
|
+
if (testConfig?.scorers && testConfig.scorers.length > 0) {
|
|
13969
|
+
await runScorers({
|
|
13970
|
+
scorers: testConfig.scorers,
|
|
13971
|
+
output: testResult,
|
|
13972
|
+
expected: testConfig.expected,
|
|
13973
|
+
input: testConfig.input,
|
|
13974
|
+
metadata: testConfig.metadata,
|
|
13975
|
+
span
|
|
13976
|
+
});
|
|
13977
|
+
}
|
|
13978
|
+
span.log({
|
|
13979
|
+
scores: {
|
|
13980
|
+
pass: 0
|
|
13981
|
+
},
|
|
13982
|
+
metadata: {
|
|
13983
|
+
error: error instanceof Error ? {
|
|
13984
|
+
message: error.message,
|
|
13985
|
+
name: error.name,
|
|
13986
|
+
stack: error.stack
|
|
13987
|
+
} : String(error)
|
|
13988
|
+
}
|
|
13989
|
+
});
|
|
13990
|
+
throw error;
|
|
13991
|
+
}
|
|
13992
|
+
return testResult;
|
|
13993
|
+
},
|
|
13994
|
+
{
|
|
13995
|
+
name,
|
|
13996
|
+
spanAttributes: {
|
|
13997
|
+
type: "task" /* TASK */
|
|
13998
|
+
},
|
|
13999
|
+
event: testConfig ? {
|
|
14000
|
+
input: testConfig.input,
|
|
14001
|
+
expected: testConfig.expected,
|
|
14002
|
+
metadata: testConfig.metadata,
|
|
14003
|
+
tags: testConfig.tags
|
|
14004
|
+
} : void 0
|
|
14005
|
+
}
|
|
14006
|
+
);
|
|
14007
|
+
passed = true;
|
|
14008
|
+
return result;
|
|
14009
|
+
} catch (error) {
|
|
14010
|
+
passed = false;
|
|
14011
|
+
throw error;
|
|
14012
|
+
} finally {
|
|
14013
|
+
const duration = performance.now() - startTime;
|
|
14014
|
+
if (experimentContext) {
|
|
14015
|
+
if (passed) {
|
|
14016
|
+
experimentContext.passed = (experimentContext.passed ?? 0) + 1;
|
|
14017
|
+
} else {
|
|
14018
|
+
experimentContext.failed = (experimentContext.failed ?? 0) + 1;
|
|
14019
|
+
}
|
|
14020
|
+
}
|
|
14021
|
+
if (config.onProgress) {
|
|
14022
|
+
config.onProgress({
|
|
14023
|
+
type: "test_complete",
|
|
14024
|
+
testName: name,
|
|
14025
|
+
passed,
|
|
14026
|
+
duration
|
|
14027
|
+
});
|
|
14028
|
+
}
|
|
14029
|
+
}
|
|
14030
|
+
};
|
|
14031
|
+
return testFn(
|
|
14032
|
+
name,
|
|
14033
|
+
hasVitestOptions ? vitestOptions : void 0,
|
|
14034
|
+
testImplementation
|
|
14035
|
+
);
|
|
14036
|
+
};
|
|
14037
|
+
return wrapped;
|
|
14038
|
+
};
|
|
14039
|
+
const wrappedTest = wrapBare(originalTest);
|
|
14040
|
+
wrappedTest.skip = wrapBare(originalTest.skip);
|
|
14041
|
+
wrappedTest.only = wrapBare(originalTest.only);
|
|
14042
|
+
wrappedTest.concurrent = wrapBare(originalTest.concurrent);
|
|
14043
|
+
if (originalTest.todo) wrappedTest.todo = originalTest.todo;
|
|
14044
|
+
if (originalTest.each) wrappedTest.each = originalTest.each;
|
|
14045
|
+
return wrappedTest;
|
|
14046
|
+
}
|
|
14047
|
+
function wrapDescribe(originalDescribe, config, afterAll) {
|
|
14048
|
+
const wrapBare = (describeFn) => {
|
|
14049
|
+
const wrapped = function(suiteName, factory) {
|
|
14050
|
+
return describeFn(suiteName, () => {
|
|
14051
|
+
const contextManager = getVitestContextManager();
|
|
14052
|
+
let context = null;
|
|
14053
|
+
const getOrCreateContext = () => {
|
|
14054
|
+
if (!context) {
|
|
14055
|
+
const projectName = config.projectName || suiteName;
|
|
14056
|
+
const experiment = initExperiment(projectName, {
|
|
14057
|
+
experiment: `${suiteName}-${(/* @__PURE__ */ new Date()).toISOString()}`
|
|
14058
|
+
});
|
|
14059
|
+
context = contextManager.createChildContext(void 0, experiment);
|
|
14060
|
+
}
|
|
14061
|
+
return context;
|
|
14062
|
+
};
|
|
14063
|
+
const lazyContext = {
|
|
14064
|
+
get dataset() {
|
|
14065
|
+
return getOrCreateContext().dataset;
|
|
14066
|
+
},
|
|
14067
|
+
get experiment() {
|
|
14068
|
+
return getOrCreateContext().experiment;
|
|
14069
|
+
},
|
|
14070
|
+
get datasetExamples() {
|
|
14071
|
+
return getOrCreateContext().datasetExamples;
|
|
14072
|
+
},
|
|
14073
|
+
get parent() {
|
|
14074
|
+
return getOrCreateContext().parent;
|
|
14075
|
+
},
|
|
14076
|
+
get flushPromise() {
|
|
14077
|
+
return getOrCreateContext().flushPromise;
|
|
14078
|
+
},
|
|
14079
|
+
set flushPromise(value) {
|
|
14080
|
+
if (context) context.flushPromise = value;
|
|
14081
|
+
},
|
|
14082
|
+
get flushResolved() {
|
|
14083
|
+
return getOrCreateContext().flushResolved;
|
|
14084
|
+
},
|
|
14085
|
+
set flushResolved(value) {
|
|
14086
|
+
if (context) context.flushResolved = value;
|
|
14087
|
+
}
|
|
14088
|
+
};
|
|
14089
|
+
if (config.onProgress) {
|
|
14090
|
+
config.onProgress({ type: "suite_start", suiteName });
|
|
14091
|
+
}
|
|
14092
|
+
contextManager.setContext(lazyContext);
|
|
14093
|
+
factory();
|
|
14094
|
+
if (afterAll && (config.displaySummary ?? true)) {
|
|
14095
|
+
afterAll(async () => {
|
|
14096
|
+
await flushExperimentWithSync(context, config);
|
|
14097
|
+
if (config.onProgress) {
|
|
14098
|
+
config.onProgress({
|
|
14099
|
+
type: "suite_complete",
|
|
14100
|
+
suiteName,
|
|
14101
|
+
passed: context?.passed ?? 0,
|
|
14102
|
+
failed: context?.failed ?? 0
|
|
14103
|
+
});
|
|
14104
|
+
}
|
|
14105
|
+
});
|
|
14106
|
+
}
|
|
14107
|
+
});
|
|
14108
|
+
};
|
|
14109
|
+
return wrapped;
|
|
14110
|
+
};
|
|
14111
|
+
const wrappedDescribe = wrapBare(originalDescribe);
|
|
14112
|
+
wrappedDescribe.skip = wrapBare(originalDescribe.skip);
|
|
14113
|
+
wrappedDescribe.only = wrapBare(originalDescribe.only);
|
|
14114
|
+
wrappedDescribe.concurrent = wrapBare(originalDescribe.concurrent);
|
|
14115
|
+
if (originalDescribe.todo) wrappedDescribe.todo = originalDescribe.todo;
|
|
14116
|
+
if (originalDescribe.each)
|
|
14117
|
+
wrappedDescribe.each = originalDescribe.each;
|
|
14118
|
+
return wrappedDescribe;
|
|
14119
|
+
}
|
|
14120
|
+
|
|
14121
|
+
// src/wrappers/vitest/expect-wrapper.ts
|
|
14122
|
+
function proxyAssertion(assertion, value, key, span) {
|
|
14123
|
+
return new Proxy(assertion, {
|
|
14124
|
+
get(target, prop, receiver) {
|
|
14125
|
+
const original = Reflect.get(target, prop, receiver);
|
|
14126
|
+
if (typeof original === "function") {
|
|
14127
|
+
return function(...args) {
|
|
14128
|
+
let result;
|
|
14129
|
+
try {
|
|
14130
|
+
result = original.apply(target, args);
|
|
14131
|
+
} catch (err) {
|
|
14132
|
+
span.log({ output: { [key]: value }, scores: { [key]: 0 } });
|
|
14133
|
+
throw err;
|
|
14134
|
+
}
|
|
14135
|
+
if (result !== null && typeof result === "object" && "then" in result && typeof Reflect.get(result, "then") === "function") {
|
|
14136
|
+
return Promise.resolve(result).then(
|
|
14137
|
+
(v) => {
|
|
14138
|
+
span.log({ output: { [key]: value }, scores: { [key]: 1 } });
|
|
14139
|
+
return v;
|
|
14140
|
+
},
|
|
14141
|
+
(err) => {
|
|
14142
|
+
span.log({ output: { [key]: value }, scores: { [key]: 0 } });
|
|
14143
|
+
throw err;
|
|
14144
|
+
}
|
|
14145
|
+
);
|
|
14146
|
+
}
|
|
14147
|
+
span.log({ output: { [key]: value }, scores: { [key]: 1 } });
|
|
14148
|
+
return result;
|
|
14149
|
+
};
|
|
14150
|
+
}
|
|
14151
|
+
if (original !== null && typeof original === "object") {
|
|
14152
|
+
return proxyAssertion(original, value, key, span);
|
|
14153
|
+
}
|
|
14154
|
+
return original;
|
|
14155
|
+
}
|
|
14156
|
+
});
|
|
14157
|
+
}
|
|
14158
|
+
function wrapExpect(originalExpect) {
|
|
14159
|
+
const wrapped = function(value, message) {
|
|
14160
|
+
if (message === void 0) {
|
|
14161
|
+
return originalExpect(value);
|
|
14162
|
+
}
|
|
14163
|
+
const assertion = originalExpect(value, message);
|
|
14164
|
+
const span = currentSpan();
|
|
14165
|
+
if (!span) {
|
|
14166
|
+
return assertion;
|
|
14167
|
+
}
|
|
14168
|
+
if (assertion === null || typeof assertion !== "object") return assertion;
|
|
14169
|
+
return proxyAssertion(assertion, value, message, span);
|
|
14170
|
+
};
|
|
14171
|
+
return Object.assign(wrapped, originalExpect);
|
|
14172
|
+
}
|
|
14173
|
+
|
|
14174
|
+
// src/wrappers/vitest/index.ts
|
|
14175
|
+
function wrapVitest(vitestMethods, config = {}) {
|
|
14176
|
+
if (!vitestMethods.test) {
|
|
14177
|
+
throw new Error(
|
|
14178
|
+
"Braintrust: vitestMethods.test is required. Please pass in the test function from vitest."
|
|
14179
|
+
);
|
|
14180
|
+
}
|
|
14181
|
+
if (!vitestMethods.describe) {
|
|
14182
|
+
throw new Error(
|
|
14183
|
+
"Braintrust: vitestMethods.describe is required. Please pass in the describe function from vitest."
|
|
14184
|
+
);
|
|
14185
|
+
}
|
|
14186
|
+
if (!vitestMethods.expect) {
|
|
14187
|
+
throw new Error(
|
|
14188
|
+
"Braintrust: vitestMethods.expect is required. Please pass in the expect function from vitest."
|
|
14189
|
+
);
|
|
14190
|
+
}
|
|
14191
|
+
const wrappedTest = wrapTest(vitestMethods.test, config);
|
|
14192
|
+
const wrappedDescribe = wrapDescribe(
|
|
14193
|
+
vitestMethods.describe,
|
|
14194
|
+
config,
|
|
14195
|
+
vitestMethods.afterAll
|
|
14196
|
+
);
|
|
14197
|
+
return {
|
|
14198
|
+
test: wrappedTest,
|
|
14199
|
+
it: wrappedTest,
|
|
14200
|
+
expect: wrapExpect(vitestMethods.expect),
|
|
14201
|
+
describe: wrappedDescribe,
|
|
14202
|
+
beforeAll: vitestMethods.beforeAll || (() => {
|
|
14203
|
+
}),
|
|
14204
|
+
afterAll: vitestMethods.afterAll || (() => {
|
|
14205
|
+
}),
|
|
14206
|
+
beforeEach: vitestMethods.beforeEach,
|
|
14207
|
+
afterEach: vitestMethods.afterEach,
|
|
14208
|
+
logOutputs: (outputs) => {
|
|
14209
|
+
const span = currentSpan();
|
|
14210
|
+
if (!span) {
|
|
14211
|
+
console.warn(
|
|
14212
|
+
"Braintrust: No active span. logOutputs() must be called within a wrapped test."
|
|
14213
|
+
);
|
|
14214
|
+
return;
|
|
14215
|
+
}
|
|
14216
|
+
span.log({ output: outputs });
|
|
14217
|
+
},
|
|
14218
|
+
logFeedback: (feedback) => {
|
|
14219
|
+
const span = currentSpan();
|
|
14220
|
+
if (!span) {
|
|
14221
|
+
console.warn(
|
|
14222
|
+
"Braintrust: No active span. logFeedback() must be called within a wrapped test."
|
|
14223
|
+
);
|
|
14224
|
+
return;
|
|
14225
|
+
}
|
|
14226
|
+
span.log({
|
|
14227
|
+
scores: {
|
|
14228
|
+
[feedback.name]: feedback.score
|
|
14229
|
+
},
|
|
14230
|
+
metadata: feedback.metadata
|
|
14231
|
+
});
|
|
14232
|
+
},
|
|
14233
|
+
getCurrentSpan: () => {
|
|
14234
|
+
return currentSpan();
|
|
14235
|
+
},
|
|
14236
|
+
flushExperiment: async (options) => {
|
|
14237
|
+
const ctx = getExperimentContext();
|
|
14238
|
+
if (!ctx) {
|
|
14239
|
+
console.warn(
|
|
14240
|
+
"Braintrust: No experiment context found. Make sure you're using bt.describe() and calling flushExperiment() within an afterAll() hook."
|
|
14241
|
+
);
|
|
14242
|
+
return;
|
|
14243
|
+
}
|
|
14244
|
+
const shouldDisplaySummary = options?.displaySummary ?? config.displaySummary ?? true;
|
|
14245
|
+
let summary;
|
|
14246
|
+
if (shouldDisplaySummary) {
|
|
14247
|
+
try {
|
|
14248
|
+
summary = await ctx.experiment.summarize();
|
|
14249
|
+
} catch (error) {
|
|
14250
|
+
console.warn(
|
|
14251
|
+
"Braintrust: Failed to generate experiment summary:",
|
|
14252
|
+
error
|
|
14253
|
+
);
|
|
14254
|
+
}
|
|
14255
|
+
}
|
|
14256
|
+
await ctx.experiment.flush();
|
|
14257
|
+
if (summary && shouldDisplaySummary) {
|
|
14258
|
+
console.log(formatExperimentSummary(summary));
|
|
14259
|
+
}
|
|
14260
|
+
}
|
|
14261
|
+
};
|
|
14262
|
+
}
|
|
14263
|
+
|
|
13694
14264
|
// src/graph-framework.ts
|
|
13695
14265
|
var graph_framework_exports = {};
|
|
13696
14266
|
__export(graph_framework_exports, {
|
|
@@ -16707,6 +17277,2482 @@ var evaluatorDefinitionsSchema = z12.record(
|
|
|
16707
17277
|
evaluatorDefinitionSchema
|
|
16708
17278
|
);
|
|
16709
17279
|
|
|
17280
|
+
// src/instrumentation/core/plugin.ts
|
|
17281
|
+
import { tracingChannel } from "dc-browser";
|
|
17282
|
+
|
|
17283
|
+
// src/instrumentation/core/stream-patcher.ts
|
|
17284
|
+
function isAsyncIterable4(value) {
|
|
17285
|
+
return value !== null && typeof value === "object" && Symbol.asyncIterator in value && typeof value[Symbol.asyncIterator] === "function";
|
|
17286
|
+
}
|
|
17287
|
+
function patchStreamIfNeeded(stream, options) {
|
|
17288
|
+
if (!isAsyncIterable4(stream)) {
|
|
17289
|
+
return stream;
|
|
17290
|
+
}
|
|
17291
|
+
if (Object.isFrozen(stream) || Object.isSealed(stream)) {
|
|
17292
|
+
console.warn(
|
|
17293
|
+
"Cannot patch frozen/sealed stream. Stream output will not be collected."
|
|
17294
|
+
);
|
|
17295
|
+
return stream;
|
|
17296
|
+
}
|
|
17297
|
+
const originalIteratorFn = stream[Symbol.asyncIterator];
|
|
17298
|
+
if (originalIteratorFn.__braintrust_patched) {
|
|
17299
|
+
return stream;
|
|
17300
|
+
}
|
|
17301
|
+
try {
|
|
17302
|
+
const patchedIteratorFn = function() {
|
|
17303
|
+
const iterator = originalIteratorFn.call(this);
|
|
17304
|
+
const originalNext = iterator.next.bind(iterator);
|
|
17305
|
+
const chunks = [];
|
|
17306
|
+
let completed = false;
|
|
17307
|
+
iterator.next = async function(...args) {
|
|
17308
|
+
try {
|
|
17309
|
+
const result = await originalNext(...args);
|
|
17310
|
+
if (result.done) {
|
|
17311
|
+
if (!completed) {
|
|
17312
|
+
completed = true;
|
|
17313
|
+
try {
|
|
17314
|
+
options.onComplete(chunks);
|
|
17315
|
+
} catch (error) {
|
|
17316
|
+
console.error("Error in stream onComplete handler:", error);
|
|
17317
|
+
}
|
|
17318
|
+
}
|
|
17319
|
+
} else {
|
|
17320
|
+
const chunk = result.value;
|
|
17321
|
+
const shouldCollect = options.shouldCollect ? options.shouldCollect(chunk) : true;
|
|
17322
|
+
if (shouldCollect) {
|
|
17323
|
+
chunks.push(chunk);
|
|
17324
|
+
if (options.onChunk) {
|
|
17325
|
+
try {
|
|
17326
|
+
options.onChunk(chunk);
|
|
17327
|
+
} catch (error) {
|
|
17328
|
+
console.error("Error in stream onChunk handler:", error);
|
|
17329
|
+
}
|
|
17330
|
+
}
|
|
17331
|
+
}
|
|
17332
|
+
}
|
|
17333
|
+
return result;
|
|
17334
|
+
} catch (error) {
|
|
17335
|
+
if (!completed) {
|
|
17336
|
+
completed = true;
|
|
17337
|
+
if (options.onError) {
|
|
17338
|
+
try {
|
|
17339
|
+
options.onError(error, chunks);
|
|
17340
|
+
} catch (handlerError) {
|
|
17341
|
+
console.error("Error in stream onError handler:", handlerError);
|
|
17342
|
+
}
|
|
17343
|
+
}
|
|
17344
|
+
}
|
|
17345
|
+
throw error;
|
|
17346
|
+
}
|
|
17347
|
+
};
|
|
17348
|
+
if (iterator.return) {
|
|
17349
|
+
const originalReturn = iterator.return.bind(iterator);
|
|
17350
|
+
iterator.return = async function(...args) {
|
|
17351
|
+
if (!completed) {
|
|
17352
|
+
completed = true;
|
|
17353
|
+
try {
|
|
17354
|
+
options.onComplete(chunks);
|
|
17355
|
+
} catch (error) {
|
|
17356
|
+
console.error("Error in stream onComplete handler:", error);
|
|
17357
|
+
}
|
|
17358
|
+
}
|
|
17359
|
+
return originalReturn(...args);
|
|
17360
|
+
};
|
|
17361
|
+
}
|
|
17362
|
+
if (iterator.throw) {
|
|
17363
|
+
const originalThrow = iterator.throw.bind(iterator);
|
|
17364
|
+
iterator.throw = async function(...args) {
|
|
17365
|
+
if (!completed) {
|
|
17366
|
+
completed = true;
|
|
17367
|
+
const error = args[0];
|
|
17368
|
+
if (options.onError) {
|
|
17369
|
+
try {
|
|
17370
|
+
options.onError(error, chunks);
|
|
17371
|
+
} catch (handlerError) {
|
|
17372
|
+
console.error("Error in stream onError handler:", handlerError);
|
|
17373
|
+
}
|
|
17374
|
+
}
|
|
17375
|
+
}
|
|
17376
|
+
return originalThrow(...args);
|
|
17377
|
+
};
|
|
17378
|
+
}
|
|
17379
|
+
return iterator;
|
|
17380
|
+
};
|
|
17381
|
+
patchedIteratorFn.__braintrust_patched = true;
|
|
17382
|
+
stream[Symbol.asyncIterator] = patchedIteratorFn;
|
|
17383
|
+
return stream;
|
|
17384
|
+
} catch (error) {
|
|
17385
|
+
console.warn("Failed to patch stream:", error);
|
|
17386
|
+
return stream;
|
|
17387
|
+
}
|
|
17388
|
+
}
|
|
17389
|
+
|
|
17390
|
+
// src/instrumentation/core/plugin.ts
|
|
17391
|
+
var BasePlugin = class {
|
|
17392
|
+
enabled = false;
|
|
17393
|
+
unsubscribers = [];
|
|
17394
|
+
/**
|
|
17395
|
+
* Enables the plugin. Must be called before the plugin will receive events.
|
|
17396
|
+
*/
|
|
17397
|
+
enable() {
|
|
17398
|
+
if (this.enabled) {
|
|
17399
|
+
return;
|
|
17400
|
+
}
|
|
17401
|
+
this.enabled = true;
|
|
17402
|
+
this.onEnable();
|
|
17403
|
+
}
|
|
17404
|
+
/**
|
|
17405
|
+
* Disables the plugin. After this, the plugin will no longer receive events.
|
|
17406
|
+
*/
|
|
17407
|
+
disable() {
|
|
17408
|
+
if (!this.enabled) {
|
|
17409
|
+
return;
|
|
17410
|
+
}
|
|
17411
|
+
this.enabled = false;
|
|
17412
|
+
this.onDisable();
|
|
17413
|
+
}
|
|
17414
|
+
/**
|
|
17415
|
+
* Helper to subscribe to a channel with raw handlers.
|
|
17416
|
+
*
|
|
17417
|
+
* @param channelName - The channel name to subscribe to
|
|
17418
|
+
* @param handlers - Event handlers
|
|
17419
|
+
*/
|
|
17420
|
+
subscribe(channelName, handlers) {
|
|
17421
|
+
const channel = tracingChannel(channelName);
|
|
17422
|
+
channel.subscribe(handlers);
|
|
17423
|
+
}
|
|
17424
|
+
/**
|
|
17425
|
+
* Subscribe to a channel for async methods (non-streaming).
|
|
17426
|
+
* Creates a span and logs input/output/metrics.
|
|
17427
|
+
*/
|
|
17428
|
+
subscribeToChannel(channelName, config) {
|
|
17429
|
+
const channel = tracingChannel(channelName);
|
|
17430
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
17431
|
+
const handlers = {
|
|
17432
|
+
start: (event) => {
|
|
17433
|
+
const span = startSpan({
|
|
17434
|
+
name: config.name,
|
|
17435
|
+
spanAttributes: {
|
|
17436
|
+
type: config.type
|
|
17437
|
+
}
|
|
17438
|
+
});
|
|
17439
|
+
const startTime = getCurrentUnixTimestamp();
|
|
17440
|
+
spans.set(event, { span, startTime });
|
|
17441
|
+
try {
|
|
17442
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
17443
|
+
span.log({
|
|
17444
|
+
input,
|
|
17445
|
+
metadata
|
|
17446
|
+
});
|
|
17447
|
+
} catch (error) {
|
|
17448
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
17449
|
+
}
|
|
17450
|
+
},
|
|
17451
|
+
asyncEnd: (event) => {
|
|
17452
|
+
const spanData = spans.get(event);
|
|
17453
|
+
if (!spanData) {
|
|
17454
|
+
return;
|
|
17455
|
+
}
|
|
17456
|
+
const { span, startTime } = spanData;
|
|
17457
|
+
try {
|
|
17458
|
+
const output = config.extractOutput(event.result);
|
|
17459
|
+
const metrics = config.extractMetrics(event.result, startTime);
|
|
17460
|
+
span.log({
|
|
17461
|
+
output,
|
|
17462
|
+
metrics
|
|
17463
|
+
});
|
|
17464
|
+
} catch (error) {
|
|
17465
|
+
console.error(`Error extracting output for ${channelName}:`, error);
|
|
17466
|
+
} finally {
|
|
17467
|
+
span.end();
|
|
17468
|
+
spans.delete(event);
|
|
17469
|
+
}
|
|
17470
|
+
},
|
|
17471
|
+
error: (event) => {
|
|
17472
|
+
const spanData = spans.get(event);
|
|
17473
|
+
if (!spanData) {
|
|
17474
|
+
return;
|
|
17475
|
+
}
|
|
17476
|
+
const { span } = spanData;
|
|
17477
|
+
span.log({
|
|
17478
|
+
error: event.error.message
|
|
17479
|
+
});
|
|
17480
|
+
span.end();
|
|
17481
|
+
spans.delete(event);
|
|
17482
|
+
}
|
|
17483
|
+
};
|
|
17484
|
+
channel.subscribe(handlers);
|
|
17485
|
+
this.unsubscribers.push(() => {
|
|
17486
|
+
channel.unsubscribe(handlers);
|
|
17487
|
+
});
|
|
17488
|
+
}
|
|
17489
|
+
/**
|
|
17490
|
+
* Subscribe to a channel for async methods that may return streams.
|
|
17491
|
+
* Handles both streaming and non-streaming responses.
|
|
17492
|
+
*/
|
|
17493
|
+
subscribeToStreamingChannel(channelName, config) {
|
|
17494
|
+
const channel = tracingChannel(channelName);
|
|
17495
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
17496
|
+
const handlers = {
|
|
17497
|
+
start: (event) => {
|
|
17498
|
+
const span = startSpan({
|
|
17499
|
+
name: config.name,
|
|
17500
|
+
spanAttributes: {
|
|
17501
|
+
type: config.type
|
|
17502
|
+
}
|
|
17503
|
+
});
|
|
17504
|
+
const startTime = getCurrentUnixTimestamp();
|
|
17505
|
+
spans.set(event, { span, startTime });
|
|
17506
|
+
try {
|
|
17507
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
17508
|
+
span.log({
|
|
17509
|
+
input,
|
|
17510
|
+
metadata
|
|
17511
|
+
});
|
|
17512
|
+
} catch (error) {
|
|
17513
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
17514
|
+
}
|
|
17515
|
+
},
|
|
17516
|
+
asyncEnd: (event) => {
|
|
17517
|
+
const spanData = spans.get(event);
|
|
17518
|
+
if (!spanData) {
|
|
17519
|
+
return;
|
|
17520
|
+
}
|
|
17521
|
+
const { span, startTime } = spanData;
|
|
17522
|
+
if (isAsyncIterable4(event.result)) {
|
|
17523
|
+
patchStreamIfNeeded(event.result, {
|
|
17524
|
+
onComplete: (chunks) => {
|
|
17525
|
+
try {
|
|
17526
|
+
let output;
|
|
17527
|
+
let metrics;
|
|
17528
|
+
if (config.aggregateChunks) {
|
|
17529
|
+
const aggregated = config.aggregateChunks(chunks);
|
|
17530
|
+
output = aggregated.output;
|
|
17531
|
+
metrics = aggregated.metrics;
|
|
17532
|
+
} else {
|
|
17533
|
+
output = config.extractOutput(chunks);
|
|
17534
|
+
metrics = config.extractMetrics(chunks, startTime);
|
|
17535
|
+
}
|
|
17536
|
+
if (!metrics.time_to_first_token && chunks.length > 0) {
|
|
17537
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
17538
|
+
}
|
|
17539
|
+
span.log({
|
|
17540
|
+
output,
|
|
17541
|
+
metrics
|
|
17542
|
+
});
|
|
17543
|
+
} catch (error) {
|
|
17544
|
+
console.error(
|
|
17545
|
+
`Error extracting output for ${channelName}:`,
|
|
17546
|
+
error
|
|
17547
|
+
);
|
|
17548
|
+
} finally {
|
|
17549
|
+
span.end();
|
|
17550
|
+
}
|
|
17551
|
+
},
|
|
17552
|
+
onError: (error) => {
|
|
17553
|
+
span.log({
|
|
17554
|
+
error: error.message
|
|
17555
|
+
});
|
|
17556
|
+
span.end();
|
|
17557
|
+
}
|
|
17558
|
+
});
|
|
17559
|
+
} else {
|
|
17560
|
+
try {
|
|
17561
|
+
const output = config.extractOutput(event.result);
|
|
17562
|
+
const metrics = config.extractMetrics(event.result, startTime);
|
|
17563
|
+
span.log({
|
|
17564
|
+
output,
|
|
17565
|
+
metrics
|
|
17566
|
+
});
|
|
17567
|
+
} catch (error) {
|
|
17568
|
+
console.error(`Error extracting output for ${channelName}:`, error);
|
|
17569
|
+
} finally {
|
|
17570
|
+
span.end();
|
|
17571
|
+
spans.delete(event);
|
|
17572
|
+
}
|
|
17573
|
+
}
|
|
17574
|
+
},
|
|
17575
|
+
error: (event) => {
|
|
17576
|
+
const spanData = spans.get(event);
|
|
17577
|
+
if (!spanData) {
|
|
17578
|
+
return;
|
|
17579
|
+
}
|
|
17580
|
+
const { span } = spanData;
|
|
17581
|
+
span.log({
|
|
17582
|
+
error: event.error.message
|
|
17583
|
+
});
|
|
17584
|
+
span.end();
|
|
17585
|
+
spans.delete(event);
|
|
17586
|
+
}
|
|
17587
|
+
};
|
|
17588
|
+
channel.subscribe(handlers);
|
|
17589
|
+
this.unsubscribers.push(() => {
|
|
17590
|
+
channel.unsubscribe(handlers);
|
|
17591
|
+
});
|
|
17592
|
+
}
|
|
17593
|
+
/**
|
|
17594
|
+
* Subscribe to a channel for sync methods that return event-based streams.
|
|
17595
|
+
* Used for methods like beta.chat.completions.stream() and responses.stream().
|
|
17596
|
+
*/
|
|
17597
|
+
subscribeToSyncStreamChannel(channelName, config) {
|
|
17598
|
+
const channel = tracingChannel(channelName);
|
|
17599
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
17600
|
+
const handlers = {
|
|
17601
|
+
start: (event) => {
|
|
17602
|
+
const span = startSpan({
|
|
17603
|
+
name: config.name,
|
|
17604
|
+
spanAttributes: {
|
|
17605
|
+
type: config.type
|
|
17606
|
+
}
|
|
17607
|
+
});
|
|
17608
|
+
const startTime = getCurrentUnixTimestamp();
|
|
17609
|
+
spans.set(event, { span, startTime });
|
|
17610
|
+
try {
|
|
17611
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
17612
|
+
span.log({
|
|
17613
|
+
input,
|
|
17614
|
+
metadata
|
|
17615
|
+
});
|
|
17616
|
+
} catch (error) {
|
|
17617
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
17618
|
+
}
|
|
17619
|
+
},
|
|
17620
|
+
end: (event) => {
|
|
17621
|
+
const spanData = spans.get(event);
|
|
17622
|
+
if (!spanData) {
|
|
17623
|
+
return;
|
|
17624
|
+
}
|
|
17625
|
+
const { span, startTime } = spanData;
|
|
17626
|
+
const stream = event.result;
|
|
17627
|
+
if (!stream || typeof stream.on !== "function") {
|
|
17628
|
+
span.end();
|
|
17629
|
+
spans.delete(event);
|
|
17630
|
+
return;
|
|
17631
|
+
}
|
|
17632
|
+
let first = true;
|
|
17633
|
+
stream.on("chunk", (chunk) => {
|
|
17634
|
+
if (first) {
|
|
17635
|
+
const now2 = getCurrentUnixTimestamp();
|
|
17636
|
+
span.log({
|
|
17637
|
+
metrics: {
|
|
17638
|
+
time_to_first_token: now2 - startTime
|
|
17639
|
+
}
|
|
17640
|
+
});
|
|
17641
|
+
first = false;
|
|
17642
|
+
}
|
|
17643
|
+
});
|
|
17644
|
+
stream.on("chatCompletion", (completion) => {
|
|
17645
|
+
try {
|
|
17646
|
+
span.log({
|
|
17647
|
+
output: completion.choices
|
|
17648
|
+
});
|
|
17649
|
+
} catch (error) {
|
|
17650
|
+
console.error(
|
|
17651
|
+
`Error extracting chatCompletion for ${channelName}:`,
|
|
17652
|
+
error
|
|
17653
|
+
);
|
|
17654
|
+
}
|
|
17655
|
+
});
|
|
17656
|
+
stream.on("event", (streamEvent) => {
|
|
17657
|
+
if (config.extractFromEvent) {
|
|
17658
|
+
try {
|
|
17659
|
+
if (first) {
|
|
17660
|
+
const now2 = getCurrentUnixTimestamp();
|
|
17661
|
+
span.log({
|
|
17662
|
+
metrics: {
|
|
17663
|
+
time_to_first_token: now2 - startTime
|
|
17664
|
+
}
|
|
17665
|
+
});
|
|
17666
|
+
first = false;
|
|
17667
|
+
}
|
|
17668
|
+
const extracted = config.extractFromEvent(streamEvent);
|
|
17669
|
+
if (extracted && Object.keys(extracted).length > 0) {
|
|
17670
|
+
span.log(extracted);
|
|
17671
|
+
}
|
|
17672
|
+
} catch (error) {
|
|
17673
|
+
console.error(
|
|
17674
|
+
`Error extracting event for ${channelName}:`,
|
|
17675
|
+
error
|
|
17676
|
+
);
|
|
17677
|
+
}
|
|
17678
|
+
}
|
|
17679
|
+
});
|
|
17680
|
+
stream.on("end", () => {
|
|
17681
|
+
span.end();
|
|
17682
|
+
spans.delete(event);
|
|
17683
|
+
});
|
|
17684
|
+
},
|
|
17685
|
+
error: (event) => {
|
|
17686
|
+
const spanData = spans.get(event);
|
|
17687
|
+
if (!spanData) {
|
|
17688
|
+
return;
|
|
17689
|
+
}
|
|
17690
|
+
const { span } = spanData;
|
|
17691
|
+
span.log({
|
|
17692
|
+
error: event.error.message
|
|
17693
|
+
});
|
|
17694
|
+
span.end();
|
|
17695
|
+
spans.delete(event);
|
|
17696
|
+
}
|
|
17697
|
+
};
|
|
17698
|
+
channel.subscribe(handlers);
|
|
17699
|
+
this.unsubscribers.push(() => {
|
|
17700
|
+
channel.unsubscribe(handlers);
|
|
17701
|
+
});
|
|
17702
|
+
}
|
|
17703
|
+
};
|
|
17704
|
+
|
|
17705
|
+
// src/instrumentation/plugins/openai-plugin.ts
|
|
17706
|
+
var OpenAIPlugin = class extends BasePlugin {
|
|
17707
|
+
constructor() {
|
|
17708
|
+
super();
|
|
17709
|
+
}
|
|
17710
|
+
onEnable() {
|
|
17711
|
+
this.subscribeToStreamingChannel(
|
|
17712
|
+
"orchestrion:openai:chat.completions.create",
|
|
17713
|
+
{
|
|
17714
|
+
name: "Chat Completion",
|
|
17715
|
+
type: "llm" /* LLM */,
|
|
17716
|
+
extractInput: (args) => {
|
|
17717
|
+
const params = args[0] || {};
|
|
17718
|
+
const { messages, ...metadata } = params;
|
|
17719
|
+
return {
|
|
17720
|
+
input: processInputAttachments(messages),
|
|
17721
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17722
|
+
};
|
|
17723
|
+
},
|
|
17724
|
+
extractOutput: (result) => {
|
|
17725
|
+
return result?.choices;
|
|
17726
|
+
},
|
|
17727
|
+
extractMetrics: (result, startTime) => {
|
|
17728
|
+
const metrics = parseMetricsFromUsage3(result?.usage);
|
|
17729
|
+
if (startTime) {
|
|
17730
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
17731
|
+
}
|
|
17732
|
+
return metrics;
|
|
17733
|
+
},
|
|
17734
|
+
aggregateChunks: aggregateChatCompletionChunks
|
|
17735
|
+
}
|
|
17736
|
+
);
|
|
17737
|
+
this.subscribeToChannel("orchestrion:openai:embeddings.create", {
|
|
17738
|
+
name: "Embedding",
|
|
17739
|
+
type: "llm" /* LLM */,
|
|
17740
|
+
extractInput: (args) => {
|
|
17741
|
+
const params = args[0] || {};
|
|
17742
|
+
const { input, ...metadata } = params;
|
|
17743
|
+
return {
|
|
17744
|
+
input,
|
|
17745
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17746
|
+
};
|
|
17747
|
+
},
|
|
17748
|
+
extractOutput: (result) => {
|
|
17749
|
+
return result?.data?.map((d) => d.embedding);
|
|
17750
|
+
},
|
|
17751
|
+
extractMetrics: (result) => {
|
|
17752
|
+
return parseMetricsFromUsage3(result?.usage);
|
|
17753
|
+
}
|
|
17754
|
+
});
|
|
17755
|
+
this.subscribeToStreamingChannel(
|
|
17756
|
+
"orchestrion:openai:beta.chat.completions.parse",
|
|
17757
|
+
{
|
|
17758
|
+
name: "Chat Completion",
|
|
17759
|
+
type: "llm" /* LLM */,
|
|
17760
|
+
extractInput: (args) => {
|
|
17761
|
+
const params = args[0] || {};
|
|
17762
|
+
const { messages, ...metadata } = params;
|
|
17763
|
+
return {
|
|
17764
|
+
input: processInputAttachments(messages),
|
|
17765
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17766
|
+
};
|
|
17767
|
+
},
|
|
17768
|
+
extractOutput: (result) => {
|
|
17769
|
+
return result?.choices;
|
|
17770
|
+
},
|
|
17771
|
+
extractMetrics: (result, startTime) => {
|
|
17772
|
+
const metrics = parseMetricsFromUsage3(result?.usage);
|
|
17773
|
+
if (startTime) {
|
|
17774
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
17775
|
+
}
|
|
17776
|
+
return metrics;
|
|
17777
|
+
},
|
|
17778
|
+
aggregateChunks: aggregateChatCompletionChunks
|
|
17779
|
+
}
|
|
17780
|
+
);
|
|
17781
|
+
this.subscribeToSyncStreamChannel(
|
|
17782
|
+
"orchestrion:openai:beta.chat.completions.stream",
|
|
17783
|
+
{
|
|
17784
|
+
name: "Chat Completion",
|
|
17785
|
+
type: "llm" /* LLM */,
|
|
17786
|
+
extractInput: (args) => {
|
|
17787
|
+
const params = args[0] || {};
|
|
17788
|
+
const { messages, ...metadata } = params;
|
|
17789
|
+
return {
|
|
17790
|
+
input: processInputAttachments(messages),
|
|
17791
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17792
|
+
};
|
|
17793
|
+
}
|
|
17794
|
+
}
|
|
17795
|
+
);
|
|
17796
|
+
this.subscribeToChannel("orchestrion:openai:moderations.create", {
|
|
17797
|
+
name: "Moderation",
|
|
17798
|
+
type: "llm" /* LLM */,
|
|
17799
|
+
extractInput: (args) => {
|
|
17800
|
+
const params = args[0] || {};
|
|
17801
|
+
const { input, ...metadata } = params;
|
|
17802
|
+
return {
|
|
17803
|
+
input,
|
|
17804
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17805
|
+
};
|
|
17806
|
+
},
|
|
17807
|
+
extractOutput: (result) => {
|
|
17808
|
+
return result?.results;
|
|
17809
|
+
},
|
|
17810
|
+
extractMetrics: () => {
|
|
17811
|
+
return {};
|
|
17812
|
+
}
|
|
17813
|
+
});
|
|
17814
|
+
this.subscribeToStreamingChannel("orchestrion:openai:responses.create", {
|
|
17815
|
+
name: "openai.responses.create",
|
|
17816
|
+
type: "llm" /* LLM */,
|
|
17817
|
+
extractInput: (args) => {
|
|
17818
|
+
const params = args[0] || {};
|
|
17819
|
+
const { input, ...metadata } = params;
|
|
17820
|
+
return {
|
|
17821
|
+
input: processInputAttachments(input),
|
|
17822
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17823
|
+
};
|
|
17824
|
+
},
|
|
17825
|
+
extractOutput: (result) => {
|
|
17826
|
+
return processImagesInOutput2(result?.output);
|
|
17827
|
+
},
|
|
17828
|
+
extractMetrics: (result, startTime) => {
|
|
17829
|
+
const metrics = parseMetricsFromUsage3(result?.usage);
|
|
17830
|
+
if (startTime) {
|
|
17831
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
17832
|
+
}
|
|
17833
|
+
return metrics;
|
|
17834
|
+
}
|
|
17835
|
+
});
|
|
17836
|
+
this.subscribeToSyncStreamChannel("orchestrion:openai:responses.stream", {
|
|
17837
|
+
name: "openai.responses.stream",
|
|
17838
|
+
type: "llm" /* LLM */,
|
|
17839
|
+
extractInput: (args) => {
|
|
17840
|
+
const params = args[0] || {};
|
|
17841
|
+
const { input, ...metadata } = params;
|
|
17842
|
+
return {
|
|
17843
|
+
input: processInputAttachments(input),
|
|
17844
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17845
|
+
};
|
|
17846
|
+
},
|
|
17847
|
+
extractFromEvent: (event) => {
|
|
17848
|
+
if (!event || !event.type || !event.response) {
|
|
17849
|
+
return {};
|
|
17850
|
+
}
|
|
17851
|
+
const response = event.response;
|
|
17852
|
+
if (event.type === "response.completed") {
|
|
17853
|
+
const data = {};
|
|
17854
|
+
if (response?.output !== void 0) {
|
|
17855
|
+
data.output = processImagesInOutput2(response.output);
|
|
17856
|
+
}
|
|
17857
|
+
if (response) {
|
|
17858
|
+
const { usage: _usage, output: _output, ...metadata } = response;
|
|
17859
|
+
if (Object.keys(metadata).length > 0) {
|
|
17860
|
+
data.metadata = metadata;
|
|
17861
|
+
}
|
|
17862
|
+
}
|
|
17863
|
+
data.metrics = parseMetricsFromUsage3(response?.usage);
|
|
17864
|
+
return data;
|
|
17865
|
+
}
|
|
17866
|
+
return {};
|
|
17867
|
+
}
|
|
17868
|
+
});
|
|
17869
|
+
this.subscribeToStreamingChannel("orchestrion:openai:responses.parse", {
|
|
17870
|
+
name: "openai.responses.parse",
|
|
17871
|
+
type: "llm" /* LLM */,
|
|
17872
|
+
extractInput: (args) => {
|
|
17873
|
+
const params = args[0] || {};
|
|
17874
|
+
const { input, ...metadata } = params;
|
|
17875
|
+
return {
|
|
17876
|
+
input: processInputAttachments(input),
|
|
17877
|
+
metadata: { ...metadata, provider: "openai" }
|
|
17878
|
+
};
|
|
17879
|
+
},
|
|
17880
|
+
extractOutput: (result) => {
|
|
17881
|
+
return processImagesInOutput2(result?.output);
|
|
17882
|
+
},
|
|
17883
|
+
extractMetrics: (result, startTime) => {
|
|
17884
|
+
const metrics = parseMetricsFromUsage3(result?.usage);
|
|
17885
|
+
if (startTime) {
|
|
17886
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
17887
|
+
}
|
|
17888
|
+
return metrics;
|
|
17889
|
+
}
|
|
17890
|
+
});
|
|
17891
|
+
}
|
|
17892
|
+
onDisable() {
|
|
17893
|
+
}
|
|
17894
|
+
};
|
|
17895
|
+
var TOKEN_NAME_MAP2 = {
|
|
17896
|
+
input_tokens: "prompt_tokens",
|
|
17897
|
+
output_tokens: "completion_tokens",
|
|
17898
|
+
total_tokens: "tokens"
|
|
17899
|
+
};
|
|
17900
|
+
var TOKEN_PREFIX_MAP2 = {
|
|
17901
|
+
input: "prompt",
|
|
17902
|
+
output: "completion"
|
|
17903
|
+
};
|
|
17904
|
+
function parseMetricsFromUsage3(usage) {
|
|
17905
|
+
if (!usage) {
|
|
17906
|
+
return {};
|
|
17907
|
+
}
|
|
17908
|
+
const metrics = {};
|
|
17909
|
+
for (const [oai_name, value] of Object.entries(usage)) {
|
|
17910
|
+
if (typeof value === "number") {
|
|
17911
|
+
const metricName = TOKEN_NAME_MAP2[oai_name] || oai_name;
|
|
17912
|
+
metrics[metricName] = value;
|
|
17913
|
+
} else if (oai_name.endsWith("_tokens_details")) {
|
|
17914
|
+
if (!isObject(value)) {
|
|
17915
|
+
continue;
|
|
17916
|
+
}
|
|
17917
|
+
const rawPrefix = oai_name.slice(0, -"_tokens_details".length);
|
|
17918
|
+
const prefix = TOKEN_PREFIX_MAP2[rawPrefix] || rawPrefix;
|
|
17919
|
+
for (const [key, n] of Object.entries(value)) {
|
|
17920
|
+
if (typeof n !== "number") {
|
|
17921
|
+
continue;
|
|
17922
|
+
}
|
|
17923
|
+
const metricName = `${prefix}_${key}`;
|
|
17924
|
+
metrics[metricName] = n;
|
|
17925
|
+
}
|
|
17926
|
+
}
|
|
17927
|
+
}
|
|
17928
|
+
return metrics;
|
|
17929
|
+
}
|
|
17930
|
+
function processImagesInOutput2(output) {
|
|
17931
|
+
if (Array.isArray(output)) {
|
|
17932
|
+
return output.map(processImagesInOutput2);
|
|
17933
|
+
}
|
|
17934
|
+
if (isObject(output)) {
|
|
17935
|
+
if (output.type === "image_generation_call" && output.result && typeof output.result === "string") {
|
|
17936
|
+
const fileExtension = output.output_format || "png";
|
|
17937
|
+
const contentType = `image/${fileExtension}`;
|
|
17938
|
+
const baseFilename = output.revised_prompt && typeof output.revised_prompt === "string" ? output.revised_prompt.slice(0, 50).replace(/[^a-zA-Z0-9]/g, "_") : "generated_image";
|
|
17939
|
+
const filename = `${baseFilename}.${fileExtension}`;
|
|
17940
|
+
const binaryString = atob(output.result);
|
|
17941
|
+
const bytes = new Uint8Array(binaryString.length);
|
|
17942
|
+
for (let i = 0; i < binaryString.length; i++) {
|
|
17943
|
+
bytes[i] = binaryString.charCodeAt(i);
|
|
17944
|
+
}
|
|
17945
|
+
const blob = new Blob([bytes], { type: contentType });
|
|
17946
|
+
const attachment = new Attachment({
|
|
17947
|
+
data: blob,
|
|
17948
|
+
filename,
|
|
17949
|
+
contentType
|
|
17950
|
+
});
|
|
17951
|
+
return {
|
|
17952
|
+
...output,
|
|
17953
|
+
result: attachment
|
|
17954
|
+
};
|
|
17955
|
+
}
|
|
17956
|
+
}
|
|
17957
|
+
return output;
|
|
17958
|
+
}
|
|
17959
|
+
function aggregateChatCompletionChunks(chunks) {
|
|
17960
|
+
let role = void 0;
|
|
17961
|
+
let content = void 0;
|
|
17962
|
+
let tool_calls = void 0;
|
|
17963
|
+
let finish_reason = void 0;
|
|
17964
|
+
let metrics = {};
|
|
17965
|
+
for (const chunk of chunks) {
|
|
17966
|
+
if (chunk.usage) {
|
|
17967
|
+
metrics = {
|
|
17968
|
+
...metrics,
|
|
17969
|
+
...parseMetricsFromUsage3(chunk.usage)
|
|
17970
|
+
};
|
|
17971
|
+
}
|
|
17972
|
+
const delta = chunk.choices?.[0]?.delta;
|
|
17973
|
+
if (!delta) {
|
|
17974
|
+
continue;
|
|
17975
|
+
}
|
|
17976
|
+
if (!role && delta.role) {
|
|
17977
|
+
role = delta.role;
|
|
17978
|
+
}
|
|
17979
|
+
if (delta.finish_reason) {
|
|
17980
|
+
finish_reason = delta.finish_reason;
|
|
17981
|
+
}
|
|
17982
|
+
if (delta.content) {
|
|
17983
|
+
content = (content || "") + delta.content;
|
|
17984
|
+
}
|
|
17985
|
+
if (delta.tool_calls) {
|
|
17986
|
+
const toolDelta = delta.tool_calls[0];
|
|
17987
|
+
if (!tool_calls || toolDelta.id && tool_calls[tool_calls.length - 1].id !== toolDelta.id) {
|
|
17988
|
+
tool_calls = [
|
|
17989
|
+
...tool_calls || [],
|
|
17990
|
+
{
|
|
17991
|
+
id: toolDelta.id,
|
|
17992
|
+
type: toolDelta.type,
|
|
17993
|
+
function: toolDelta.function
|
|
17994
|
+
}
|
|
17995
|
+
];
|
|
17996
|
+
} else {
|
|
17997
|
+
tool_calls[tool_calls.length - 1].function.arguments += toolDelta.function.arguments;
|
|
17998
|
+
}
|
|
17999
|
+
}
|
|
18000
|
+
}
|
|
18001
|
+
return {
|
|
18002
|
+
metrics,
|
|
18003
|
+
output: [
|
|
18004
|
+
{
|
|
18005
|
+
index: 0,
|
|
18006
|
+
message: {
|
|
18007
|
+
role,
|
|
18008
|
+
content,
|
|
18009
|
+
tool_calls
|
|
18010
|
+
},
|
|
18011
|
+
logprobs: null,
|
|
18012
|
+
finish_reason
|
|
18013
|
+
}
|
|
18014
|
+
]
|
|
18015
|
+
};
|
|
18016
|
+
}
|
|
18017
|
+
|
|
18018
|
+
// src/instrumentation/plugins/anthropic-plugin.ts
|
|
18019
|
+
import { tracingChannel as tracingChannel2 } from "dc-browser";
|
|
18020
|
+
var AnthropicPlugin = class extends BasePlugin {
|
|
18021
|
+
unsubscribers = [];
|
|
18022
|
+
onEnable() {
|
|
18023
|
+
this.subscribeToAnthropicChannels();
|
|
18024
|
+
}
|
|
18025
|
+
onDisable() {
|
|
18026
|
+
for (const unsubscribe of this.unsubscribers) {
|
|
18027
|
+
unsubscribe();
|
|
18028
|
+
}
|
|
18029
|
+
this.unsubscribers = [];
|
|
18030
|
+
}
|
|
18031
|
+
subscribeToAnthropicChannels() {
|
|
18032
|
+
this.subscribeToStreamingChannel("orchestrion:anthropic:messages.create", {
|
|
18033
|
+
name: "anthropic.messages.create",
|
|
18034
|
+
type: "llm" /* LLM */,
|
|
18035
|
+
extractInput: (args) => {
|
|
18036
|
+
const params = args[0] || {};
|
|
18037
|
+
const input = coalesceInput2(params.messages || [], params.system);
|
|
18038
|
+
const metadata = filterFrom2(params, ["messages", "system"]);
|
|
18039
|
+
return {
|
|
18040
|
+
input: processAttachmentsInInput3(input),
|
|
18041
|
+
metadata: { ...metadata, provider: "anthropic" }
|
|
18042
|
+
};
|
|
18043
|
+
},
|
|
18044
|
+
extractOutput: (result) => {
|
|
18045
|
+
return result ? { role: result.role, content: result.content } : null;
|
|
18046
|
+
},
|
|
18047
|
+
extractMetrics: (result, startTime) => {
|
|
18048
|
+
const metrics = parseMetricsFromUsage4(result?.usage);
|
|
18049
|
+
if (startTime) {
|
|
18050
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18051
|
+
}
|
|
18052
|
+
const finalized = finalizeAnthropicTokens(metrics);
|
|
18053
|
+
return Object.fromEntries(
|
|
18054
|
+
Object.entries(finalized).filter(([, v]) => v !== void 0)
|
|
18055
|
+
);
|
|
18056
|
+
},
|
|
18057
|
+
extractMetadata: (result) => {
|
|
18058
|
+
const metadata = {};
|
|
18059
|
+
const metas = ["stop_reason", "stop_sequence"];
|
|
18060
|
+
for (const m of metas) {
|
|
18061
|
+
if (result?.[m] !== void 0) {
|
|
18062
|
+
metadata[m] = result[m];
|
|
18063
|
+
}
|
|
18064
|
+
}
|
|
18065
|
+
return metadata;
|
|
18066
|
+
},
|
|
18067
|
+
aggregateChunks: aggregateAnthropicStreamChunks,
|
|
18068
|
+
isStreaming: (args) => {
|
|
18069
|
+
return args[0]?.stream === true;
|
|
18070
|
+
}
|
|
18071
|
+
});
|
|
18072
|
+
this.subscribeToStreamingChannel(
|
|
18073
|
+
"orchestrion:anthropic:beta.messages.create",
|
|
18074
|
+
{
|
|
18075
|
+
name: "anthropic.beta.messages.create",
|
|
18076
|
+
type: "llm" /* LLM */,
|
|
18077
|
+
extractInput: (args) => {
|
|
18078
|
+
const params = args[0] || {};
|
|
18079
|
+
const input = coalesceInput2(params.messages || [], params.system);
|
|
18080
|
+
const metadata = filterFrom2(params, ["messages", "system"]);
|
|
18081
|
+
return {
|
|
18082
|
+
input: processAttachmentsInInput3(input),
|
|
18083
|
+
metadata: { ...metadata, provider: "anthropic" }
|
|
18084
|
+
};
|
|
18085
|
+
},
|
|
18086
|
+
extractOutput: (result) => {
|
|
18087
|
+
return result ? { role: result.role, content: result.content } : null;
|
|
18088
|
+
},
|
|
18089
|
+
extractMetrics: (result, startTime) => {
|
|
18090
|
+
const metrics = parseMetricsFromUsage4(result?.usage);
|
|
18091
|
+
if (startTime) {
|
|
18092
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18093
|
+
}
|
|
18094
|
+
const finalized = finalizeAnthropicTokens(metrics);
|
|
18095
|
+
return Object.fromEntries(
|
|
18096
|
+
Object.entries(finalized).filter(([, v]) => v !== void 0)
|
|
18097
|
+
);
|
|
18098
|
+
},
|
|
18099
|
+
extractMetadata: (result) => {
|
|
18100
|
+
const metadata = {};
|
|
18101
|
+
const metas = ["stop_reason", "stop_sequence"];
|
|
18102
|
+
for (const m of metas) {
|
|
18103
|
+
if (result?.[m] !== void 0) {
|
|
18104
|
+
metadata[m] = result[m];
|
|
18105
|
+
}
|
|
18106
|
+
}
|
|
18107
|
+
return metadata;
|
|
18108
|
+
},
|
|
18109
|
+
aggregateChunks: aggregateAnthropicStreamChunks,
|
|
18110
|
+
isStreaming: (args) => {
|
|
18111
|
+
return args[0]?.stream === true;
|
|
18112
|
+
}
|
|
18113
|
+
}
|
|
18114
|
+
);
|
|
18115
|
+
}
|
|
18116
|
+
/**
|
|
18117
|
+
* Subscribe to a channel for async methods that may return streams.
|
|
18118
|
+
* Handles both streaming and non-streaming responses based on the stream parameter.
|
|
18119
|
+
*/
|
|
18120
|
+
subscribeToStreamingChannel(channelName, config) {
|
|
18121
|
+
const channel = tracingChannel2(channelName);
|
|
18122
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
18123
|
+
const handlers = {
|
|
18124
|
+
start: (event) => {
|
|
18125
|
+
const span = startSpan({
|
|
18126
|
+
name: config.name,
|
|
18127
|
+
spanAttributes: {
|
|
18128
|
+
type: config.type
|
|
18129
|
+
}
|
|
18130
|
+
});
|
|
18131
|
+
const startTime = getCurrentUnixTimestamp();
|
|
18132
|
+
spans.set(event, { span, startTime });
|
|
18133
|
+
try {
|
|
18134
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
18135
|
+
span.log({
|
|
18136
|
+
input,
|
|
18137
|
+
metadata
|
|
18138
|
+
});
|
|
18139
|
+
} catch (error) {
|
|
18140
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
18141
|
+
}
|
|
18142
|
+
},
|
|
18143
|
+
asyncEnd: (event) => {
|
|
18144
|
+
const spanData = spans.get(event);
|
|
18145
|
+
if (!spanData) {
|
|
18146
|
+
return;
|
|
18147
|
+
}
|
|
18148
|
+
const { span, startTime } = spanData;
|
|
18149
|
+
const isStreaming = config.isStreaming ? config.isStreaming(event.arguments) : isAsyncIterable4(event.result);
|
|
18150
|
+
if (isStreaming && isAsyncIterable4(event.result)) {
|
|
18151
|
+
patchStreamIfNeeded(event.result, {
|
|
18152
|
+
onComplete: (chunks) => {
|
|
18153
|
+
try {
|
|
18154
|
+
let output;
|
|
18155
|
+
let metrics;
|
|
18156
|
+
let metadata = {};
|
|
18157
|
+
if (config.aggregateChunks) {
|
|
18158
|
+
const aggregated = config.aggregateChunks(chunks);
|
|
18159
|
+
output = aggregated.output;
|
|
18160
|
+
metrics = aggregated.metrics;
|
|
18161
|
+
metadata = aggregated.metadata || {};
|
|
18162
|
+
} else {
|
|
18163
|
+
output = config.extractOutput(chunks);
|
|
18164
|
+
metrics = config.extractMetrics(chunks, startTime);
|
|
18165
|
+
if (config.extractMetadata) {
|
|
18166
|
+
metadata = config.extractMetadata(chunks);
|
|
18167
|
+
}
|
|
18168
|
+
}
|
|
18169
|
+
if (!metrics.time_to_first_token && chunks.length > 0) {
|
|
18170
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18171
|
+
}
|
|
18172
|
+
span.log({
|
|
18173
|
+
output,
|
|
18174
|
+
metrics,
|
|
18175
|
+
metadata
|
|
18176
|
+
});
|
|
18177
|
+
} catch (error) {
|
|
18178
|
+
console.error(
|
|
18179
|
+
`Error extracting output for ${channelName}:`,
|
|
18180
|
+
error
|
|
18181
|
+
);
|
|
18182
|
+
} finally {
|
|
18183
|
+
span.end();
|
|
18184
|
+
}
|
|
18185
|
+
},
|
|
18186
|
+
onError: (error) => {
|
|
18187
|
+
span.log({
|
|
18188
|
+
error: error.message
|
|
18189
|
+
});
|
|
18190
|
+
span.end();
|
|
18191
|
+
}
|
|
18192
|
+
});
|
|
18193
|
+
} else {
|
|
18194
|
+
try {
|
|
18195
|
+
const output = config.extractOutput(event.result);
|
|
18196
|
+
const metrics = config.extractMetrics(event.result, startTime);
|
|
18197
|
+
const metadata = config.extractMetadata ? config.extractMetadata(event.result) : {};
|
|
18198
|
+
span.log({
|
|
18199
|
+
output,
|
|
18200
|
+
metrics,
|
|
18201
|
+
metadata
|
|
18202
|
+
});
|
|
18203
|
+
} catch (error) {
|
|
18204
|
+
console.error(`Error extracting output for ${channelName}:`, error);
|
|
18205
|
+
} finally {
|
|
18206
|
+
span.end();
|
|
18207
|
+
spans.delete(event);
|
|
18208
|
+
}
|
|
18209
|
+
}
|
|
18210
|
+
},
|
|
18211
|
+
error: (event) => {
|
|
18212
|
+
const spanData = spans.get(event);
|
|
18213
|
+
if (!spanData) {
|
|
18214
|
+
return;
|
|
18215
|
+
}
|
|
18216
|
+
const { span } = spanData;
|
|
18217
|
+
span.log({
|
|
18218
|
+
error: event.error.message
|
|
18219
|
+
});
|
|
18220
|
+
span.end();
|
|
18221
|
+
spans.delete(event);
|
|
18222
|
+
}
|
|
18223
|
+
};
|
|
18224
|
+
channel.subscribe(handlers);
|
|
18225
|
+
this.unsubscribers.push(() => {
|
|
18226
|
+
channel.unsubscribe(handlers);
|
|
18227
|
+
});
|
|
18228
|
+
}
|
|
18229
|
+
};
|
|
18230
|
+
function parseMetricsFromUsage4(usage) {
|
|
18231
|
+
if (!usage) {
|
|
18232
|
+
return {};
|
|
18233
|
+
}
|
|
18234
|
+
const metrics = {};
|
|
18235
|
+
function saveIfExistsTo(source, target) {
|
|
18236
|
+
const value = usage[source];
|
|
18237
|
+
if (value !== void 0 && value !== null && typeof value === "number") {
|
|
18238
|
+
metrics[target] = value;
|
|
18239
|
+
}
|
|
18240
|
+
}
|
|
18241
|
+
saveIfExistsTo("input_tokens", "prompt_tokens");
|
|
18242
|
+
saveIfExistsTo("output_tokens", "completion_tokens");
|
|
18243
|
+
saveIfExistsTo("cache_read_input_tokens", "prompt_cached_tokens");
|
|
18244
|
+
saveIfExistsTo("cache_creation_input_tokens", "prompt_cache_creation_tokens");
|
|
18245
|
+
return metrics;
|
|
18246
|
+
}
|
|
18247
|
+
function aggregateAnthropicStreamChunks(chunks) {
|
|
18248
|
+
const deltas = [];
|
|
18249
|
+
let metrics = {};
|
|
18250
|
+
let metadata = {};
|
|
18251
|
+
for (const chunk of chunks) {
|
|
18252
|
+
switch (chunk?.type) {
|
|
18253
|
+
case "message_start":
|
|
18254
|
+
if (chunk.message?.usage) {
|
|
18255
|
+
const initialMetrics = parseMetricsFromUsage4(chunk.message.usage);
|
|
18256
|
+
metrics = { ...metrics, ...initialMetrics };
|
|
18257
|
+
}
|
|
18258
|
+
break;
|
|
18259
|
+
case "content_block_delta":
|
|
18260
|
+
if (chunk.delta?.type === "text_delta") {
|
|
18261
|
+
const text = chunk.delta?.text;
|
|
18262
|
+
if (text) {
|
|
18263
|
+
deltas.push(text);
|
|
18264
|
+
}
|
|
18265
|
+
}
|
|
18266
|
+
break;
|
|
18267
|
+
case "message_delta":
|
|
18268
|
+
if (chunk.usage) {
|
|
18269
|
+
const finalMetrics = parseMetricsFromUsage4(chunk.usage);
|
|
18270
|
+
metrics = { ...metrics, ...finalMetrics };
|
|
18271
|
+
}
|
|
18272
|
+
if (chunk.delta) {
|
|
18273
|
+
metadata = { ...metadata, ...chunk.delta };
|
|
18274
|
+
}
|
|
18275
|
+
break;
|
|
18276
|
+
}
|
|
18277
|
+
}
|
|
18278
|
+
const output = deltas.join("");
|
|
18279
|
+
const finalized = finalizeAnthropicTokens(metrics);
|
|
18280
|
+
const filteredMetrics = Object.fromEntries(
|
|
18281
|
+
Object.entries(finalized).filter(([, v]) => v !== void 0)
|
|
18282
|
+
);
|
|
18283
|
+
return {
|
|
18284
|
+
output,
|
|
18285
|
+
metrics: filteredMetrics,
|
|
18286
|
+
metadata
|
|
18287
|
+
};
|
|
18288
|
+
}
|
|
18289
|
+
function convertBase64ToAttachment2(source, contentType) {
|
|
18290
|
+
const mediaType = typeof source.media_type === "string" ? source.media_type : "image/png";
|
|
18291
|
+
const base64Data = source.data;
|
|
18292
|
+
if (base64Data && typeof base64Data === "string") {
|
|
18293
|
+
const binaryString = atob(base64Data);
|
|
18294
|
+
const bytes = new Uint8Array(binaryString.length);
|
|
18295
|
+
for (let i = 0; i < binaryString.length; i++) {
|
|
18296
|
+
bytes[i] = binaryString.charCodeAt(i);
|
|
18297
|
+
}
|
|
18298
|
+
const blob = new Blob([bytes], { type: mediaType });
|
|
18299
|
+
const extension = mediaType.split("/")[1] || "bin";
|
|
18300
|
+
const prefix = contentType === "document" ? "document" : "image";
|
|
18301
|
+
const filename = `${prefix}.${extension}`;
|
|
18302
|
+
const attachment = new Attachment({
|
|
18303
|
+
data: blob,
|
|
18304
|
+
filename,
|
|
18305
|
+
contentType: mediaType
|
|
18306
|
+
});
|
|
18307
|
+
return {
|
|
18308
|
+
...source,
|
|
18309
|
+
data: attachment
|
|
18310
|
+
};
|
|
18311
|
+
}
|
|
18312
|
+
return source;
|
|
18313
|
+
}
|
|
18314
|
+
function processAttachmentsInInput3(input) {
|
|
18315
|
+
if (Array.isArray(input)) {
|
|
18316
|
+
return input.map(processAttachmentsInInput3);
|
|
18317
|
+
}
|
|
18318
|
+
if (isObject(input)) {
|
|
18319
|
+
if ((input.type === "image" || input.type === "document") && isObject(input.source) && input.source.type === "base64") {
|
|
18320
|
+
return {
|
|
18321
|
+
...input,
|
|
18322
|
+
source: convertBase64ToAttachment2(input.source, input.type)
|
|
18323
|
+
};
|
|
18324
|
+
}
|
|
18325
|
+
const processed = {};
|
|
18326
|
+
for (const [key, value] of Object.entries(input)) {
|
|
18327
|
+
processed[key] = processAttachmentsInInput3(value);
|
|
18328
|
+
}
|
|
18329
|
+
return processed;
|
|
18330
|
+
}
|
|
18331
|
+
return input;
|
|
18332
|
+
}
|
|
18333
|
+
function coalesceInput2(messages, system) {
|
|
18334
|
+
const input = (messages || []).slice();
|
|
18335
|
+
if (system) {
|
|
18336
|
+
input.push({ role: "system", content: system });
|
|
18337
|
+
}
|
|
18338
|
+
return input;
|
|
18339
|
+
}
|
|
18340
|
+
function filterFrom2(obj, fieldsToRemove) {
|
|
18341
|
+
const result = {};
|
|
18342
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
18343
|
+
if (!fieldsToRemove.includes(key)) {
|
|
18344
|
+
result[key] = value;
|
|
18345
|
+
}
|
|
18346
|
+
}
|
|
18347
|
+
return result;
|
|
18348
|
+
}
|
|
18349
|
+
|
|
18350
|
+
// src/instrumentation/plugins/ai-sdk-plugin.ts
|
|
18351
|
+
import { tracingChannel as tracingChannel3 } from "dc-browser";
|
|
18352
|
+
var DEFAULT_DENY_OUTPUT_PATHS = [
|
|
18353
|
+
// v3
|
|
18354
|
+
"roundtrips[].request.body",
|
|
18355
|
+
"roundtrips[].response.headers",
|
|
18356
|
+
"rawResponse.headers",
|
|
18357
|
+
"responseMessages",
|
|
18358
|
+
// v5
|
|
18359
|
+
"request.body",
|
|
18360
|
+
"response.body",
|
|
18361
|
+
"response.headers",
|
|
18362
|
+
"steps[].request.body",
|
|
18363
|
+
"steps[].response.body",
|
|
18364
|
+
"steps[].response.headers"
|
|
18365
|
+
];
|
|
18366
|
+
var AISDKPlugin = class extends BasePlugin {
|
|
18367
|
+
unsubscribers = [];
|
|
18368
|
+
config;
|
|
18369
|
+
constructor(config = {}) {
|
|
18370
|
+
super();
|
|
18371
|
+
this.config = config;
|
|
18372
|
+
}
|
|
18373
|
+
onEnable() {
|
|
18374
|
+
this.subscribeToAISDK();
|
|
18375
|
+
}
|
|
18376
|
+
onDisable() {
|
|
18377
|
+
for (const unsubscribe of this.unsubscribers) {
|
|
18378
|
+
unsubscribe();
|
|
18379
|
+
}
|
|
18380
|
+
this.unsubscribers = [];
|
|
18381
|
+
}
|
|
18382
|
+
subscribeToAISDK() {
|
|
18383
|
+
const denyOutputPaths = this.config.denyOutputPaths || DEFAULT_DENY_OUTPUT_PATHS;
|
|
18384
|
+
this.subscribeToStreamingChannel("orchestrion:ai-sdk:generateText", {
|
|
18385
|
+
name: "generateText",
|
|
18386
|
+
type: "llm" /* LLM */,
|
|
18387
|
+
extractInput: (args) => {
|
|
18388
|
+
const params = args[0] || {};
|
|
18389
|
+
return {
|
|
18390
|
+
input: processAISDKInput(params),
|
|
18391
|
+
metadata: extractMetadataFromParams(params)
|
|
18392
|
+
};
|
|
18393
|
+
},
|
|
18394
|
+
extractOutput: (result) => {
|
|
18395
|
+
return processAISDKOutput(result, denyOutputPaths);
|
|
18396
|
+
},
|
|
18397
|
+
extractMetrics: (result, startTime) => {
|
|
18398
|
+
const metrics = extractTokenMetrics2(result);
|
|
18399
|
+
if (startTime) {
|
|
18400
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18401
|
+
}
|
|
18402
|
+
return metrics;
|
|
18403
|
+
},
|
|
18404
|
+
aggregateChunks: aggregateAISDKChunks
|
|
18405
|
+
});
|
|
18406
|
+
this.subscribeToStreamingChannel("orchestrion:ai-sdk:streamText", {
|
|
18407
|
+
name: "streamText",
|
|
18408
|
+
type: "llm" /* LLM */,
|
|
18409
|
+
extractInput: (args) => {
|
|
18410
|
+
const params = args[0] || {};
|
|
18411
|
+
return {
|
|
18412
|
+
input: processAISDKInput(params),
|
|
18413
|
+
metadata: extractMetadataFromParams(params)
|
|
18414
|
+
};
|
|
18415
|
+
},
|
|
18416
|
+
extractOutput: (result) => {
|
|
18417
|
+
return processAISDKOutput(result, denyOutputPaths);
|
|
18418
|
+
},
|
|
18419
|
+
extractMetrics: (result, startTime) => {
|
|
18420
|
+
const metrics = extractTokenMetrics2(result);
|
|
18421
|
+
if (startTime) {
|
|
18422
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18423
|
+
}
|
|
18424
|
+
return metrics;
|
|
18425
|
+
},
|
|
18426
|
+
aggregateChunks: aggregateAISDKChunks
|
|
18427
|
+
});
|
|
18428
|
+
this.subscribeToStreamingChannel("orchestrion:ai-sdk:generateObject", {
|
|
18429
|
+
name: "generateObject",
|
|
18430
|
+
type: "llm" /* LLM */,
|
|
18431
|
+
extractInput: (args) => {
|
|
18432
|
+
const params = args[0] || {};
|
|
18433
|
+
return {
|
|
18434
|
+
input: processAISDKInput(params),
|
|
18435
|
+
metadata: extractMetadataFromParams(params)
|
|
18436
|
+
};
|
|
18437
|
+
},
|
|
18438
|
+
extractOutput: (result) => {
|
|
18439
|
+
return processAISDKOutput(result, denyOutputPaths);
|
|
18440
|
+
},
|
|
18441
|
+
extractMetrics: (result, startTime) => {
|
|
18442
|
+
const metrics = extractTokenMetrics2(result);
|
|
18443
|
+
if (startTime) {
|
|
18444
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18445
|
+
}
|
|
18446
|
+
return metrics;
|
|
18447
|
+
},
|
|
18448
|
+
aggregateChunks: aggregateAISDKChunks
|
|
18449
|
+
});
|
|
18450
|
+
this.subscribeToStreamingChannel("orchestrion:ai-sdk:streamObject", {
|
|
18451
|
+
name: "streamObject",
|
|
18452
|
+
type: "llm" /* LLM */,
|
|
18453
|
+
extractInput: (args) => {
|
|
18454
|
+
const params = args[0] || {};
|
|
18455
|
+
return {
|
|
18456
|
+
input: processAISDKInput(params),
|
|
18457
|
+
metadata: extractMetadataFromParams(params)
|
|
18458
|
+
};
|
|
18459
|
+
},
|
|
18460
|
+
extractOutput: (result) => {
|
|
18461
|
+
return processAISDKOutput(result, denyOutputPaths);
|
|
18462
|
+
},
|
|
18463
|
+
extractMetrics: (result, startTime) => {
|
|
18464
|
+
const metrics = extractTokenMetrics2(result);
|
|
18465
|
+
if (startTime) {
|
|
18466
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18467
|
+
}
|
|
18468
|
+
return metrics;
|
|
18469
|
+
},
|
|
18470
|
+
aggregateChunks: aggregateAISDKChunks
|
|
18471
|
+
});
|
|
18472
|
+
this.subscribeToStreamingChannel("orchestrion:ai-sdk:Agent.generate", {
|
|
18473
|
+
name: "Agent.generate",
|
|
18474
|
+
type: "llm" /* LLM */,
|
|
18475
|
+
extractInput: (args) => {
|
|
18476
|
+
const params = args[0] || {};
|
|
18477
|
+
return {
|
|
18478
|
+
input: processAISDKInput(params),
|
|
18479
|
+
metadata: extractMetadataFromParams(params)
|
|
18480
|
+
};
|
|
18481
|
+
},
|
|
18482
|
+
extractOutput: (result) => {
|
|
18483
|
+
return processAISDKOutput(result, denyOutputPaths);
|
|
18484
|
+
},
|
|
18485
|
+
extractMetrics: (result, startTime) => {
|
|
18486
|
+
const metrics = extractTokenMetrics2(result);
|
|
18487
|
+
if (startTime) {
|
|
18488
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18489
|
+
}
|
|
18490
|
+
return metrics;
|
|
18491
|
+
},
|
|
18492
|
+
aggregateChunks: aggregateAISDKChunks
|
|
18493
|
+
});
|
|
18494
|
+
this.subscribeToStreamingChannel("orchestrion:ai-sdk:Agent.stream", {
|
|
18495
|
+
name: "Agent.stream",
|
|
18496
|
+
type: "llm" /* LLM */,
|
|
18497
|
+
extractInput: (args) => {
|
|
18498
|
+
const params = args[0] || {};
|
|
18499
|
+
return {
|
|
18500
|
+
input: processAISDKInput(params),
|
|
18501
|
+
metadata: extractMetadataFromParams(params)
|
|
18502
|
+
};
|
|
18503
|
+
},
|
|
18504
|
+
extractOutput: (result) => {
|
|
18505
|
+
return processAISDKOutput(result, denyOutputPaths);
|
|
18506
|
+
},
|
|
18507
|
+
extractMetrics: (result, startTime) => {
|
|
18508
|
+
const metrics = extractTokenMetrics2(result);
|
|
18509
|
+
if (startTime) {
|
|
18510
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18511
|
+
}
|
|
18512
|
+
return metrics;
|
|
18513
|
+
},
|
|
18514
|
+
aggregateChunks: aggregateAISDKChunks
|
|
18515
|
+
});
|
|
18516
|
+
}
|
|
18517
|
+
/**
|
|
18518
|
+
* Subscribe to a channel for async methods that may return streams.
|
|
18519
|
+
* Handles both streaming and non-streaming responses.
|
|
18520
|
+
*/
|
|
18521
|
+
subscribeToStreamingChannel(channelName, config) {
|
|
18522
|
+
const channel = tracingChannel3(channelName);
|
|
18523
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
18524
|
+
const handlers = {
|
|
18525
|
+
start: (event) => {
|
|
18526
|
+
const span = startSpan({
|
|
18527
|
+
name: config.name,
|
|
18528
|
+
spanAttributes: {
|
|
18529
|
+
type: config.type
|
|
18530
|
+
}
|
|
18531
|
+
});
|
|
18532
|
+
const startTime = getCurrentUnixTimestamp();
|
|
18533
|
+
spans.set(event, { span, startTime });
|
|
18534
|
+
try {
|
|
18535
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
18536
|
+
span.log({
|
|
18537
|
+
input,
|
|
18538
|
+
metadata
|
|
18539
|
+
});
|
|
18540
|
+
} catch (error) {
|
|
18541
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
18542
|
+
}
|
|
18543
|
+
},
|
|
18544
|
+
asyncEnd: (event) => {
|
|
18545
|
+
const spanData = spans.get(event);
|
|
18546
|
+
if (!spanData) {
|
|
18547
|
+
return;
|
|
18548
|
+
}
|
|
18549
|
+
const { span, startTime } = spanData;
|
|
18550
|
+
if (isAsyncIterable4(event.result)) {
|
|
18551
|
+
patchStreamIfNeeded(event.result, {
|
|
18552
|
+
onComplete: (chunks) => {
|
|
18553
|
+
try {
|
|
18554
|
+
let output;
|
|
18555
|
+
let metrics;
|
|
18556
|
+
if (config.aggregateChunks) {
|
|
18557
|
+
const aggregated = config.aggregateChunks(chunks);
|
|
18558
|
+
output = aggregated.output;
|
|
18559
|
+
metrics = aggregated.metrics;
|
|
18560
|
+
} else {
|
|
18561
|
+
output = config.extractOutput(chunks);
|
|
18562
|
+
metrics = config.extractMetrics(chunks, startTime);
|
|
18563
|
+
}
|
|
18564
|
+
if (!metrics.time_to_first_token && chunks.length > 0) {
|
|
18565
|
+
metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime;
|
|
18566
|
+
}
|
|
18567
|
+
span.log({
|
|
18568
|
+
output,
|
|
18569
|
+
metrics
|
|
18570
|
+
});
|
|
18571
|
+
} catch (error) {
|
|
18572
|
+
console.error(
|
|
18573
|
+
`Error extracting output for ${channelName}:`,
|
|
18574
|
+
error
|
|
18575
|
+
);
|
|
18576
|
+
} finally {
|
|
18577
|
+
span.end();
|
|
18578
|
+
}
|
|
18579
|
+
},
|
|
18580
|
+
onError: (error) => {
|
|
18581
|
+
span.log({
|
|
18582
|
+
error: error.message
|
|
18583
|
+
});
|
|
18584
|
+
span.end();
|
|
18585
|
+
}
|
|
18586
|
+
});
|
|
18587
|
+
} else {
|
|
18588
|
+
try {
|
|
18589
|
+
const output = config.extractOutput(event.result);
|
|
18590
|
+
const metrics = config.extractMetrics(event.result, startTime);
|
|
18591
|
+
span.log({
|
|
18592
|
+
output,
|
|
18593
|
+
metrics
|
|
18594
|
+
});
|
|
18595
|
+
} catch (error) {
|
|
18596
|
+
console.error(`Error extracting output for ${channelName}:`, error);
|
|
18597
|
+
} finally {
|
|
18598
|
+
span.end();
|
|
18599
|
+
spans.delete(event);
|
|
18600
|
+
}
|
|
18601
|
+
}
|
|
18602
|
+
},
|
|
18603
|
+
error: (event) => {
|
|
18604
|
+
const spanData = spans.get(event);
|
|
18605
|
+
if (!spanData) {
|
|
18606
|
+
return;
|
|
18607
|
+
}
|
|
18608
|
+
const { span } = spanData;
|
|
18609
|
+
span.log({
|
|
18610
|
+
error: event.error.message
|
|
18611
|
+
});
|
|
18612
|
+
span.end();
|
|
18613
|
+
spans.delete(event);
|
|
18614
|
+
}
|
|
18615
|
+
};
|
|
18616
|
+
channel.subscribe(handlers);
|
|
18617
|
+
this.unsubscribers.push(() => {
|
|
18618
|
+
channel.unsubscribe(handlers);
|
|
18619
|
+
});
|
|
18620
|
+
}
|
|
18621
|
+
};
|
|
18622
|
+
function processAISDKInput(params) {
|
|
18623
|
+
if (!params) return params;
|
|
18624
|
+
return processInputAttachments(params);
|
|
18625
|
+
}
|
|
18626
|
+
function extractMetadataFromParams(params) {
|
|
18627
|
+
const metadata = {
|
|
18628
|
+
braintrust: {
|
|
18629
|
+
integration_name: "ai-sdk",
|
|
18630
|
+
sdk_language: "typescript"
|
|
18631
|
+
}
|
|
18632
|
+
};
|
|
18633
|
+
const { model, provider } = serializeModelWithProvider2(params.model);
|
|
18634
|
+
if (model) {
|
|
18635
|
+
metadata.model = model;
|
|
18636
|
+
}
|
|
18637
|
+
if (provider) {
|
|
18638
|
+
metadata.provider = provider;
|
|
18639
|
+
}
|
|
18640
|
+
return metadata;
|
|
18641
|
+
}
|
|
18642
|
+
function processAISDKOutput(output, denyOutputPaths) {
|
|
18643
|
+
if (!output) return output;
|
|
18644
|
+
const getterValues = extractGetterValues2(output);
|
|
18645
|
+
const merged = { ...output, ...getterValues };
|
|
18646
|
+
return omit2(merged, denyOutputPaths);
|
|
18647
|
+
}
|
|
18648
|
+
function extractTokenMetrics2(result) {
|
|
18649
|
+
const metrics = {};
|
|
18650
|
+
let usage = result?.totalUsage || result?.usage;
|
|
18651
|
+
if (!usage && result) {
|
|
18652
|
+
try {
|
|
18653
|
+
if ("totalUsage" in result && typeof result.totalUsage !== "function") {
|
|
18654
|
+
usage = result.totalUsage;
|
|
18655
|
+
} else if ("usage" in result && typeof result.usage !== "function") {
|
|
18656
|
+
usage = result.usage;
|
|
18657
|
+
}
|
|
18658
|
+
} catch {
|
|
18659
|
+
}
|
|
18660
|
+
}
|
|
18661
|
+
if (!usage) {
|
|
18662
|
+
return metrics;
|
|
18663
|
+
}
|
|
18664
|
+
const promptTokens = firstNumber2(
|
|
18665
|
+
usage.inputTokens?.total,
|
|
18666
|
+
usage.inputTokens,
|
|
18667
|
+
usage.promptTokens,
|
|
18668
|
+
usage.prompt_tokens
|
|
18669
|
+
);
|
|
18670
|
+
if (promptTokens !== void 0) {
|
|
18671
|
+
metrics.prompt_tokens = promptTokens;
|
|
18672
|
+
}
|
|
18673
|
+
const completionTokens = firstNumber2(
|
|
18674
|
+
usage.outputTokens?.total,
|
|
18675
|
+
usage.outputTokens,
|
|
18676
|
+
usage.completionTokens,
|
|
18677
|
+
usage.completion_tokens
|
|
18678
|
+
);
|
|
18679
|
+
if (completionTokens !== void 0) {
|
|
18680
|
+
metrics.completion_tokens = completionTokens;
|
|
18681
|
+
}
|
|
18682
|
+
const totalTokens = firstNumber2(
|
|
18683
|
+
usage.totalTokens,
|
|
18684
|
+
usage.tokens,
|
|
18685
|
+
usage.total_tokens
|
|
18686
|
+
);
|
|
18687
|
+
if (totalTokens !== void 0) {
|
|
18688
|
+
metrics.tokens = totalTokens;
|
|
18689
|
+
}
|
|
18690
|
+
const cost = extractCostFromResult2(result);
|
|
18691
|
+
if (cost !== void 0) {
|
|
18692
|
+
metrics.estimated_cost = cost;
|
|
18693
|
+
}
|
|
18694
|
+
return metrics;
|
|
18695
|
+
}
|
|
18696
|
+
function aggregateAISDKChunks(chunks) {
|
|
18697
|
+
const lastChunk = chunks[chunks.length - 1];
|
|
18698
|
+
const output = {};
|
|
18699
|
+
let metrics = {};
|
|
18700
|
+
if (lastChunk) {
|
|
18701
|
+
metrics = extractTokenMetrics2(lastChunk);
|
|
18702
|
+
if (lastChunk.text !== void 0) {
|
|
18703
|
+
output.text = lastChunk.text;
|
|
18704
|
+
}
|
|
18705
|
+
if (lastChunk.object !== void 0) {
|
|
18706
|
+
output.object = lastChunk.object;
|
|
18707
|
+
}
|
|
18708
|
+
if (lastChunk.finishReason !== void 0) {
|
|
18709
|
+
output.finishReason = lastChunk.finishReason;
|
|
18710
|
+
}
|
|
18711
|
+
if (lastChunk.toolCalls !== void 0) {
|
|
18712
|
+
output.toolCalls = lastChunk.toolCalls;
|
|
18713
|
+
}
|
|
18714
|
+
}
|
|
18715
|
+
return { output, metrics };
|
|
18716
|
+
}
|
|
18717
|
+
function extractGetterValues2(obj) {
|
|
18718
|
+
const getterValues = {};
|
|
18719
|
+
const getterNames = [
|
|
18720
|
+
"text",
|
|
18721
|
+
"object",
|
|
18722
|
+
"finishReason",
|
|
18723
|
+
"usage",
|
|
18724
|
+
"totalUsage",
|
|
18725
|
+
"toolCalls",
|
|
18726
|
+
"toolResults",
|
|
18727
|
+
"warnings",
|
|
18728
|
+
"experimental_providerMetadata",
|
|
18729
|
+
"providerMetadata",
|
|
18730
|
+
"rawResponse",
|
|
18731
|
+
"response"
|
|
18732
|
+
];
|
|
18733
|
+
for (const name of getterNames) {
|
|
18734
|
+
try {
|
|
18735
|
+
if (obj && name in obj && typeof obj[name] !== "function") {
|
|
18736
|
+
getterValues[name] = obj[name];
|
|
18737
|
+
}
|
|
18738
|
+
} catch {
|
|
18739
|
+
}
|
|
18740
|
+
}
|
|
18741
|
+
return getterValues;
|
|
18742
|
+
}
|
|
18743
|
+
function serializeModelWithProvider2(model) {
|
|
18744
|
+
const modelId = typeof model === "string" ? model : model?.modelId;
|
|
18745
|
+
const explicitProvider = typeof model === "object" ? model?.provider : void 0;
|
|
18746
|
+
if (!modelId) {
|
|
18747
|
+
return { model: modelId, provider: explicitProvider };
|
|
18748
|
+
}
|
|
18749
|
+
const parsed = parseGatewayModelString2(modelId);
|
|
18750
|
+
return {
|
|
18751
|
+
model: parsed.model,
|
|
18752
|
+
provider: explicitProvider || parsed.provider
|
|
18753
|
+
};
|
|
18754
|
+
}
|
|
18755
|
+
function parseGatewayModelString2(modelString) {
|
|
18756
|
+
if (!modelString || typeof modelString !== "string") {
|
|
18757
|
+
return { model: modelString };
|
|
18758
|
+
}
|
|
18759
|
+
const slashIndex = modelString.indexOf("/");
|
|
18760
|
+
if (slashIndex > 0 && slashIndex < modelString.length - 1) {
|
|
18761
|
+
return {
|
|
18762
|
+
provider: modelString.substring(0, slashIndex),
|
|
18763
|
+
model: modelString.substring(slashIndex + 1)
|
|
18764
|
+
};
|
|
18765
|
+
}
|
|
18766
|
+
return { model: modelString };
|
|
18767
|
+
}
|
|
18768
|
+
function extractCostFromResult2(result) {
|
|
18769
|
+
if (result?.steps && Array.isArray(result.steps) && result.steps.length > 0) {
|
|
18770
|
+
let totalCost = 0;
|
|
18771
|
+
let foundCost = false;
|
|
18772
|
+
for (const step of result.steps) {
|
|
18773
|
+
const gateway2 = step?.providerMetadata?.gateway;
|
|
18774
|
+
const stepCost = parseGatewayCost2(gateway2?.cost) || parseGatewayCost2(gateway2?.marketCost);
|
|
18775
|
+
if (stepCost !== void 0 && stepCost > 0) {
|
|
18776
|
+
totalCost += stepCost;
|
|
18777
|
+
foundCost = true;
|
|
18778
|
+
}
|
|
18779
|
+
}
|
|
18780
|
+
if (foundCost) {
|
|
18781
|
+
return totalCost;
|
|
18782
|
+
}
|
|
18783
|
+
}
|
|
18784
|
+
const gateway = result?.providerMetadata?.gateway;
|
|
18785
|
+
const directCost = parseGatewayCost2(gateway?.cost) || parseGatewayCost2(gateway?.marketCost);
|
|
18786
|
+
if (directCost !== void 0 && directCost > 0) {
|
|
18787
|
+
return directCost;
|
|
18788
|
+
}
|
|
18789
|
+
return void 0;
|
|
18790
|
+
}
|
|
18791
|
+
function parseGatewayCost2(cost) {
|
|
18792
|
+
if (cost === void 0 || cost === null) {
|
|
18793
|
+
return void 0;
|
|
18794
|
+
}
|
|
18795
|
+
if (typeof cost === "number") {
|
|
18796
|
+
return cost;
|
|
18797
|
+
}
|
|
18798
|
+
if (typeof cost === "string") {
|
|
18799
|
+
const parsed = parseFloat(cost);
|
|
18800
|
+
if (!isNaN(parsed)) {
|
|
18801
|
+
return parsed;
|
|
18802
|
+
}
|
|
18803
|
+
}
|
|
18804
|
+
return void 0;
|
|
18805
|
+
}
|
|
18806
|
+
function firstNumber2(...values) {
|
|
18807
|
+
for (const v of values) {
|
|
18808
|
+
if (typeof v === "number") {
|
|
18809
|
+
return v;
|
|
18810
|
+
}
|
|
18811
|
+
}
|
|
18812
|
+
return void 0;
|
|
18813
|
+
}
|
|
18814
|
+
function deepCopy2(obj) {
|
|
18815
|
+
return JSON.parse(JSON.stringify(obj));
|
|
18816
|
+
}
|
|
18817
|
+
function parsePath2(path) {
|
|
18818
|
+
const keys = [];
|
|
18819
|
+
let current = "";
|
|
18820
|
+
for (let i = 0; i < path.length; i++) {
|
|
18821
|
+
const char = path[i];
|
|
18822
|
+
if (char === ".") {
|
|
18823
|
+
if (current) {
|
|
18824
|
+
keys.push(current);
|
|
18825
|
+
current = "";
|
|
18826
|
+
}
|
|
18827
|
+
} else if (char === "[") {
|
|
18828
|
+
if (current) {
|
|
18829
|
+
keys.push(current);
|
|
18830
|
+
current = "";
|
|
18831
|
+
}
|
|
18832
|
+
let bracketContent = "";
|
|
18833
|
+
i++;
|
|
18834
|
+
while (i < path.length && path[i] !== "]") {
|
|
18835
|
+
bracketContent += path[i];
|
|
18836
|
+
i++;
|
|
18837
|
+
}
|
|
18838
|
+
if (bracketContent === "") {
|
|
18839
|
+
keys.push("[]");
|
|
18840
|
+
} else {
|
|
18841
|
+
const index = parseInt(bracketContent, 10);
|
|
18842
|
+
keys.push(isNaN(index) ? bracketContent : index);
|
|
18843
|
+
}
|
|
18844
|
+
} else {
|
|
18845
|
+
current += char;
|
|
18846
|
+
}
|
|
18847
|
+
}
|
|
18848
|
+
if (current) {
|
|
18849
|
+
keys.push(current);
|
|
18850
|
+
}
|
|
18851
|
+
return keys;
|
|
18852
|
+
}
|
|
18853
|
+
function omitAtPath2(obj, keys) {
|
|
18854
|
+
if (keys.length === 0) return;
|
|
18855
|
+
const firstKey = keys[0];
|
|
18856
|
+
const remainingKeys = keys.slice(1);
|
|
18857
|
+
if (firstKey === "[]") {
|
|
18858
|
+
if (Array.isArray(obj)) {
|
|
18859
|
+
obj.forEach((item) => {
|
|
18860
|
+
if (remainingKeys.length > 0) {
|
|
18861
|
+
omitAtPath2(item, remainingKeys);
|
|
18862
|
+
}
|
|
18863
|
+
});
|
|
18864
|
+
}
|
|
18865
|
+
} else if (remainingKeys.length === 0) {
|
|
18866
|
+
if (obj && typeof obj === "object" && firstKey in obj) {
|
|
18867
|
+
obj[firstKey] = "<omitted>";
|
|
18868
|
+
}
|
|
18869
|
+
} else {
|
|
18870
|
+
if (obj && typeof obj === "object" && firstKey in obj) {
|
|
18871
|
+
omitAtPath2(obj[firstKey], remainingKeys);
|
|
18872
|
+
}
|
|
18873
|
+
}
|
|
18874
|
+
}
|
|
18875
|
+
function omit2(obj, paths) {
|
|
18876
|
+
const result = deepCopy2(obj);
|
|
18877
|
+
for (const path of paths) {
|
|
18878
|
+
const keys = parsePath2(path);
|
|
18879
|
+
omitAtPath2(result, keys);
|
|
18880
|
+
}
|
|
18881
|
+
return result;
|
|
18882
|
+
}
|
|
18883
|
+
|
|
18884
|
+
// src/instrumentation/plugins/claude-agent-sdk-plugin.ts
|
|
18885
|
+
import { tracingChannel as tracingChannel4 } from "dc-browser";
|
|
18886
|
+
function filterSerializableOptions2(options) {
|
|
18887
|
+
const allowedKeys = [
|
|
18888
|
+
"model",
|
|
18889
|
+
"maxTurns",
|
|
18890
|
+
"cwd",
|
|
18891
|
+
"continue",
|
|
18892
|
+
"allowedTools",
|
|
18893
|
+
"disallowedTools",
|
|
18894
|
+
"additionalDirectories",
|
|
18895
|
+
"permissionMode",
|
|
18896
|
+
"debug",
|
|
18897
|
+
"apiKey",
|
|
18898
|
+
"apiKeySource",
|
|
18899
|
+
"agentName",
|
|
18900
|
+
"instructions"
|
|
18901
|
+
];
|
|
18902
|
+
const filtered = {};
|
|
18903
|
+
for (const key of allowedKeys) {
|
|
18904
|
+
if (options[key] !== void 0) {
|
|
18905
|
+
filtered[key] = options[key];
|
|
18906
|
+
}
|
|
18907
|
+
}
|
|
18908
|
+
return filtered;
|
|
18909
|
+
}
|
|
18910
|
+
function getNumberProperty3(obj, key) {
|
|
18911
|
+
if (!obj || typeof obj !== "object" || !(key in obj)) {
|
|
18912
|
+
return void 0;
|
|
18913
|
+
}
|
|
18914
|
+
const value = Reflect.get(obj, key);
|
|
18915
|
+
return typeof value === "number" ? value : void 0;
|
|
18916
|
+
}
|
|
18917
|
+
function extractUsageFromMessage(message) {
|
|
18918
|
+
const metrics = {};
|
|
18919
|
+
let usage;
|
|
18920
|
+
if (message.type === "assistant") {
|
|
18921
|
+
usage = message.message?.usage;
|
|
18922
|
+
} else if (message.type === "result") {
|
|
18923
|
+
usage = message.usage;
|
|
18924
|
+
}
|
|
18925
|
+
if (!usage || typeof usage !== "object") {
|
|
18926
|
+
return metrics;
|
|
18927
|
+
}
|
|
18928
|
+
const inputTokens = getNumberProperty3(usage, "input_tokens");
|
|
18929
|
+
if (inputTokens !== void 0) {
|
|
18930
|
+
metrics.prompt_tokens = inputTokens;
|
|
18931
|
+
}
|
|
18932
|
+
const outputTokens = getNumberProperty3(usage, "output_tokens");
|
|
18933
|
+
if (outputTokens !== void 0) {
|
|
18934
|
+
metrics.completion_tokens = outputTokens;
|
|
18935
|
+
}
|
|
18936
|
+
const cacheReadTokens = getNumberProperty3(usage, "cache_read_input_tokens") || 0;
|
|
18937
|
+
const cacheCreationTokens = getNumberProperty3(usage, "cache_creation_input_tokens") || 0;
|
|
18938
|
+
if (cacheReadTokens > 0 || cacheCreationTokens > 0) {
|
|
18939
|
+
const cacheTokens = extractAnthropicCacheTokens(
|
|
18940
|
+
cacheReadTokens,
|
|
18941
|
+
cacheCreationTokens
|
|
18942
|
+
);
|
|
18943
|
+
Object.assign(metrics, cacheTokens);
|
|
18944
|
+
}
|
|
18945
|
+
if (Object.keys(metrics).length > 0) {
|
|
18946
|
+
Object.assign(metrics, finalizeAnthropicTokens(metrics));
|
|
18947
|
+
}
|
|
18948
|
+
return metrics;
|
|
18949
|
+
}
|
|
18950
|
+
function buildLLMInput(prompt, conversationHistory) {
|
|
18951
|
+
const promptMessage = typeof prompt === "string" ? { content: prompt, role: "user" } : void 0;
|
|
18952
|
+
const inputParts = [
|
|
18953
|
+
...promptMessage ? [promptMessage] : [],
|
|
18954
|
+
...conversationHistory
|
|
18955
|
+
];
|
|
18956
|
+
return inputParts.length > 0 ? inputParts : void 0;
|
|
18957
|
+
}
|
|
18958
|
+
async function createLLMSpanForMessages(messages, prompt, conversationHistory, options, startTime, parentSpan) {
|
|
18959
|
+
if (messages.length === 0) return void 0;
|
|
18960
|
+
const lastMessage = messages[messages.length - 1];
|
|
18961
|
+
if (lastMessage.type !== "assistant" || !lastMessage.message?.usage) {
|
|
18962
|
+
return void 0;
|
|
18963
|
+
}
|
|
18964
|
+
const model = lastMessage.message.model || options.model;
|
|
18965
|
+
const usage = extractUsageFromMessage(lastMessage);
|
|
18966
|
+
const input = buildLLMInput(prompt, conversationHistory);
|
|
18967
|
+
const outputs = messages.map(
|
|
18968
|
+
(m) => m.message?.content && m.message?.role ? { content: m.message.content, role: m.message.role } : void 0
|
|
18969
|
+
).filter((c) => c !== void 0);
|
|
18970
|
+
const span = startSpan({
|
|
18971
|
+
name: "anthropic.messages.create",
|
|
18972
|
+
spanAttributes: {
|
|
18973
|
+
type: "llm" /* LLM */
|
|
18974
|
+
},
|
|
18975
|
+
startTime,
|
|
18976
|
+
parent: parentSpan
|
|
18977
|
+
});
|
|
18978
|
+
span.log({
|
|
18979
|
+
input,
|
|
18980
|
+
output: outputs,
|
|
18981
|
+
metadata: model ? { model } : void 0,
|
|
18982
|
+
metrics: usage
|
|
18983
|
+
});
|
|
18984
|
+
await span.end();
|
|
18985
|
+
return lastMessage.message?.content && lastMessage.message?.role ? { content: lastMessage.message.content, role: lastMessage.message.role } : void 0;
|
|
18986
|
+
}
|
|
18987
|
+
var ClaudeAgentSDKPlugin = class extends BasePlugin {
|
|
18988
|
+
unsubscribers = [];
|
|
18989
|
+
onEnable() {
|
|
18990
|
+
this.subscribeToQuery();
|
|
18991
|
+
}
|
|
18992
|
+
onDisable() {
|
|
18993
|
+
for (const unsubscribe of this.unsubscribers) {
|
|
18994
|
+
unsubscribe();
|
|
18995
|
+
}
|
|
18996
|
+
this.unsubscribers = [];
|
|
18997
|
+
}
|
|
18998
|
+
/**
|
|
18999
|
+
* Subscribe to the query channel for agent interactions.
|
|
19000
|
+
* Handles streaming responses and traces both the top-level agent task
|
|
19001
|
+
* and individual LLM calls.
|
|
19002
|
+
*/
|
|
19003
|
+
subscribeToQuery() {
|
|
19004
|
+
const channel = tracingChannel4("orchestrion:claude-agent-sdk:query");
|
|
19005
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
19006
|
+
const handlers = {
|
|
19007
|
+
start: (event) => {
|
|
19008
|
+
const params = event.arguments[0] ?? {};
|
|
19009
|
+
const { prompt, options = {} } = params;
|
|
19010
|
+
const span = startSpan({
|
|
19011
|
+
name: "Claude Agent",
|
|
19012
|
+
spanAttributes: {
|
|
19013
|
+
type: "task" /* TASK */
|
|
19014
|
+
}
|
|
19015
|
+
});
|
|
19016
|
+
const startTime = getCurrentUnixTimestamp();
|
|
19017
|
+
try {
|
|
19018
|
+
span.log({
|
|
19019
|
+
input: typeof prompt === "string" ? prompt : {
|
|
19020
|
+
type: "streaming",
|
|
19021
|
+
description: "AsyncIterable<SDKMessage>"
|
|
19022
|
+
},
|
|
19023
|
+
metadata: filterSerializableOptions2(options)
|
|
19024
|
+
});
|
|
19025
|
+
} catch (error) {
|
|
19026
|
+
console.error("Error extracting input for Claude Agent SDK:", error);
|
|
19027
|
+
}
|
|
19028
|
+
spans.set(event, {
|
|
19029
|
+
span,
|
|
19030
|
+
startTime,
|
|
19031
|
+
conversationHistory: [],
|
|
19032
|
+
currentMessages: [],
|
|
19033
|
+
currentMessageId: void 0,
|
|
19034
|
+
currentMessageStartTime: startTime,
|
|
19035
|
+
accumulatedOutputTokens: 0
|
|
19036
|
+
});
|
|
19037
|
+
},
|
|
19038
|
+
asyncEnd: (event) => {
|
|
19039
|
+
const spanData = spans.get(event);
|
|
19040
|
+
if (!spanData) {
|
|
19041
|
+
return;
|
|
19042
|
+
}
|
|
19043
|
+
if (isAsyncIterable4(event.result)) {
|
|
19044
|
+
patchStreamIfNeeded(event.result, {
|
|
19045
|
+
onChunk: async (message) => {
|
|
19046
|
+
const currentTime = getCurrentUnixTimestamp();
|
|
19047
|
+
const params = event.arguments[0];
|
|
19048
|
+
const { prompt, options = {} } = params;
|
|
19049
|
+
const messageId = message.message?.id;
|
|
19050
|
+
if (messageId && messageId !== spanData.currentMessageId) {
|
|
19051
|
+
if (spanData.currentMessages.length > 0) {
|
|
19052
|
+
const finalMessage = await createLLMSpanForMessages(
|
|
19053
|
+
spanData.currentMessages,
|
|
19054
|
+
prompt,
|
|
19055
|
+
spanData.conversationHistory,
|
|
19056
|
+
options,
|
|
19057
|
+
spanData.currentMessageStartTime,
|
|
19058
|
+
await spanData.span.export()
|
|
19059
|
+
);
|
|
19060
|
+
if (finalMessage) {
|
|
19061
|
+
spanData.conversationHistory.push(finalMessage);
|
|
19062
|
+
}
|
|
19063
|
+
const lastMessage = spanData.currentMessages[spanData.currentMessages.length - 1];
|
|
19064
|
+
if (lastMessage?.message?.usage) {
|
|
19065
|
+
const outputTokens = getNumberProperty3(
|
|
19066
|
+
lastMessage.message.usage,
|
|
19067
|
+
"output_tokens"
|
|
19068
|
+
) || 0;
|
|
19069
|
+
spanData.accumulatedOutputTokens += outputTokens;
|
|
19070
|
+
}
|
|
19071
|
+
spanData.currentMessages = [];
|
|
19072
|
+
}
|
|
19073
|
+
spanData.currentMessageId = messageId;
|
|
19074
|
+
spanData.currentMessageStartTime = currentTime;
|
|
19075
|
+
}
|
|
19076
|
+
if (message.type === "assistant" && message.message?.usage) {
|
|
19077
|
+
spanData.currentMessages.push(message);
|
|
19078
|
+
}
|
|
19079
|
+
if (message.type === "result" && message.usage) {
|
|
19080
|
+
const finalUsageMetrics = extractUsageFromMessage(message);
|
|
19081
|
+
if (spanData.currentMessages.length > 0 && finalUsageMetrics.completion_tokens !== void 0) {
|
|
19082
|
+
const lastMessage = spanData.currentMessages[spanData.currentMessages.length - 1];
|
|
19083
|
+
if (lastMessage?.message?.usage) {
|
|
19084
|
+
const adjustedTokens = finalUsageMetrics.completion_tokens - spanData.accumulatedOutputTokens;
|
|
19085
|
+
if (adjustedTokens >= 0) {
|
|
19086
|
+
lastMessage.message.usage.output_tokens = adjustedTokens;
|
|
19087
|
+
}
|
|
19088
|
+
}
|
|
19089
|
+
}
|
|
19090
|
+
const result_metadata = {};
|
|
19091
|
+
if (message.num_turns !== void 0) {
|
|
19092
|
+
result_metadata.num_turns = message.num_turns;
|
|
19093
|
+
}
|
|
19094
|
+
if (message.session_id !== void 0) {
|
|
19095
|
+
result_metadata.session_id = message.session_id;
|
|
19096
|
+
}
|
|
19097
|
+
if (Object.keys(result_metadata).length > 0) {
|
|
19098
|
+
spanData.span.log({
|
|
19099
|
+
metadata: result_metadata
|
|
19100
|
+
});
|
|
19101
|
+
}
|
|
19102
|
+
}
|
|
19103
|
+
},
|
|
19104
|
+
onComplete: async () => {
|
|
19105
|
+
try {
|
|
19106
|
+
const params = event.arguments[0];
|
|
19107
|
+
const { prompt, options = {} } = params;
|
|
19108
|
+
if (spanData.currentMessages.length > 0) {
|
|
19109
|
+
const finalMessage = await createLLMSpanForMessages(
|
|
19110
|
+
spanData.currentMessages,
|
|
19111
|
+
prompt,
|
|
19112
|
+
spanData.conversationHistory,
|
|
19113
|
+
options,
|
|
19114
|
+
spanData.currentMessageStartTime,
|
|
19115
|
+
await spanData.span.export()
|
|
19116
|
+
);
|
|
19117
|
+
if (finalMessage) {
|
|
19118
|
+
spanData.conversationHistory.push(finalMessage);
|
|
19119
|
+
}
|
|
19120
|
+
}
|
|
19121
|
+
spanData.span.log({
|
|
19122
|
+
output: spanData.conversationHistory.length > 0 ? spanData.conversationHistory[spanData.conversationHistory.length - 1] : void 0
|
|
19123
|
+
});
|
|
19124
|
+
} catch (error) {
|
|
19125
|
+
console.error(
|
|
19126
|
+
"Error extracting output for Claude Agent SDK:",
|
|
19127
|
+
error
|
|
19128
|
+
);
|
|
19129
|
+
} finally {
|
|
19130
|
+
spanData.span.end();
|
|
19131
|
+
spans.delete(event);
|
|
19132
|
+
}
|
|
19133
|
+
},
|
|
19134
|
+
onError: (error) => {
|
|
19135
|
+
spanData.span.log({
|
|
19136
|
+
error: error.message
|
|
19137
|
+
});
|
|
19138
|
+
spanData.span.end();
|
|
19139
|
+
spans.delete(event);
|
|
19140
|
+
}
|
|
19141
|
+
});
|
|
19142
|
+
} else {
|
|
19143
|
+
try {
|
|
19144
|
+
spanData.span.log({
|
|
19145
|
+
output: event.result
|
|
19146
|
+
});
|
|
19147
|
+
} catch (error) {
|
|
19148
|
+
console.error(
|
|
19149
|
+
"Error extracting output for Claude Agent SDK:",
|
|
19150
|
+
error
|
|
19151
|
+
);
|
|
19152
|
+
} finally {
|
|
19153
|
+
spanData.span.end();
|
|
19154
|
+
spans.delete(event);
|
|
19155
|
+
}
|
|
19156
|
+
}
|
|
19157
|
+
},
|
|
19158
|
+
error: (event) => {
|
|
19159
|
+
const spanData = spans.get(event);
|
|
19160
|
+
if (!spanData) {
|
|
19161
|
+
return;
|
|
19162
|
+
}
|
|
19163
|
+
const { span } = spanData;
|
|
19164
|
+
span.log({
|
|
19165
|
+
error: event.error.message
|
|
19166
|
+
});
|
|
19167
|
+
span.end();
|
|
19168
|
+
spans.delete(event);
|
|
19169
|
+
}
|
|
19170
|
+
};
|
|
19171
|
+
channel.subscribe(handlers);
|
|
19172
|
+
this.unsubscribers.push(() => {
|
|
19173
|
+
channel.unsubscribe(handlers);
|
|
19174
|
+
});
|
|
19175
|
+
}
|
|
19176
|
+
};
|
|
19177
|
+
|
|
19178
|
+
// src/instrumentation/plugins/google-genai-plugin.ts
|
|
19179
|
+
import { tracingChannel as tracingChannel5 } from "dc-browser";
|
|
19180
|
+
var GoogleGenAIPlugin = class extends BasePlugin {
|
|
19181
|
+
unsubscribers = [];
|
|
19182
|
+
onEnable() {
|
|
19183
|
+
this.subscribeToGoogleGenAIChannels();
|
|
19184
|
+
}
|
|
19185
|
+
onDisable() {
|
|
19186
|
+
for (const unsubscribe of this.unsubscribers) {
|
|
19187
|
+
unsubscribe();
|
|
19188
|
+
}
|
|
19189
|
+
this.unsubscribers = [];
|
|
19190
|
+
}
|
|
19191
|
+
subscribeToGoogleGenAIChannels() {
|
|
19192
|
+
this.subscribeToChannel("orchestrion:google-genai:models.generateContent", {
|
|
19193
|
+
name: "google-genai.generateContent",
|
|
19194
|
+
type: "llm" /* LLM */,
|
|
19195
|
+
extractInput: (args) => {
|
|
19196
|
+
const params = args[0] || {};
|
|
19197
|
+
const input = serializeInput2(params);
|
|
19198
|
+
const metadata = extractMetadata2(params);
|
|
19199
|
+
return {
|
|
19200
|
+
input,
|
|
19201
|
+
metadata: { ...metadata, provider: "google-genai" }
|
|
19202
|
+
};
|
|
19203
|
+
},
|
|
19204
|
+
extractOutput: (result) => {
|
|
19205
|
+
return result;
|
|
19206
|
+
},
|
|
19207
|
+
extractMetrics: (result, startTime) => {
|
|
19208
|
+
return extractGenerateContentMetrics2(result, startTime);
|
|
19209
|
+
}
|
|
19210
|
+
});
|
|
19211
|
+
this.subscribeToGoogleStreamingChannel(
|
|
19212
|
+
"orchestrion:google-genai:models.generateContentStream",
|
|
19213
|
+
{
|
|
19214
|
+
name: "google-genai.generateContentStream",
|
|
19215
|
+
type: "llm" /* LLM */,
|
|
19216
|
+
extractInput: (args) => {
|
|
19217
|
+
const params = args[0] || {};
|
|
19218
|
+
const input = serializeInput2(params);
|
|
19219
|
+
const metadata = extractMetadata2(params);
|
|
19220
|
+
return {
|
|
19221
|
+
input,
|
|
19222
|
+
metadata: { ...metadata, provider: "google-genai" }
|
|
19223
|
+
};
|
|
19224
|
+
},
|
|
19225
|
+
aggregateChunks: aggregateGenerateContentChunks2
|
|
19226
|
+
}
|
|
19227
|
+
);
|
|
19228
|
+
}
|
|
19229
|
+
subscribeToChannel(channelName, config) {
|
|
19230
|
+
const channel = tracingChannel5(channelName);
|
|
19231
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
19232
|
+
const handlers = {
|
|
19233
|
+
start: (event) => {
|
|
19234
|
+
const span = startSpan({
|
|
19235
|
+
name: config.name,
|
|
19236
|
+
spanAttributes: {
|
|
19237
|
+
type: config.type
|
|
19238
|
+
}
|
|
19239
|
+
});
|
|
19240
|
+
const startTime = getCurrentUnixTimestamp();
|
|
19241
|
+
spans.set(event, { span, startTime });
|
|
19242
|
+
try {
|
|
19243
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
19244
|
+
span.log({
|
|
19245
|
+
input,
|
|
19246
|
+
metadata
|
|
19247
|
+
});
|
|
19248
|
+
} catch (error) {
|
|
19249
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
19250
|
+
}
|
|
19251
|
+
},
|
|
19252
|
+
asyncEnd: (event) => {
|
|
19253
|
+
const spanData = spans.get(event);
|
|
19254
|
+
if (!spanData) {
|
|
19255
|
+
return;
|
|
19256
|
+
}
|
|
19257
|
+
const { span, startTime } = spanData;
|
|
19258
|
+
try {
|
|
19259
|
+
const output = config.extractOutput(event.result);
|
|
19260
|
+
const metrics = config.extractMetrics(event.result, startTime);
|
|
19261
|
+
span.log({
|
|
19262
|
+
output,
|
|
19263
|
+
metrics
|
|
19264
|
+
});
|
|
19265
|
+
} catch (error) {
|
|
19266
|
+
console.error(`Error extracting output for ${channelName}:`, error);
|
|
19267
|
+
} finally {
|
|
19268
|
+
span.end();
|
|
19269
|
+
spans.delete(event);
|
|
19270
|
+
}
|
|
19271
|
+
},
|
|
19272
|
+
error: (event) => {
|
|
19273
|
+
const spanData = spans.get(event);
|
|
19274
|
+
if (!spanData) {
|
|
19275
|
+
return;
|
|
19276
|
+
}
|
|
19277
|
+
const { span } = spanData;
|
|
19278
|
+
span.log({
|
|
19279
|
+
error: event.error.message
|
|
19280
|
+
});
|
|
19281
|
+
span.end();
|
|
19282
|
+
spans.delete(event);
|
|
19283
|
+
}
|
|
19284
|
+
};
|
|
19285
|
+
channel.subscribe(handlers);
|
|
19286
|
+
this.unsubscribers.push(() => {
|
|
19287
|
+
channel.unsubscribe(handlers);
|
|
19288
|
+
});
|
|
19289
|
+
}
|
|
19290
|
+
subscribeToGoogleStreamingChannel(channelName, config) {
|
|
19291
|
+
const channel = tracingChannel5(channelName);
|
|
19292
|
+
const spans = /* @__PURE__ */ new WeakMap();
|
|
19293
|
+
const handlers = {
|
|
19294
|
+
start: (event) => {
|
|
19295
|
+
const span = startSpan({
|
|
19296
|
+
name: config.name,
|
|
19297
|
+
spanAttributes: {
|
|
19298
|
+
type: config.type
|
|
19299
|
+
}
|
|
19300
|
+
});
|
|
19301
|
+
const startTime = getCurrentUnixTimestamp();
|
|
19302
|
+
spans.set(event, { span, startTime });
|
|
19303
|
+
try {
|
|
19304
|
+
const { input, metadata } = config.extractInput(event.arguments);
|
|
19305
|
+
span.log({
|
|
19306
|
+
input,
|
|
19307
|
+
metadata
|
|
19308
|
+
});
|
|
19309
|
+
} catch (error) {
|
|
19310
|
+
console.error(`Error extracting input for ${channelName}:`, error);
|
|
19311
|
+
}
|
|
19312
|
+
},
|
|
19313
|
+
asyncEnd: (event) => {
|
|
19314
|
+
const spanData = spans.get(event);
|
|
19315
|
+
if (!spanData) {
|
|
19316
|
+
return;
|
|
19317
|
+
}
|
|
19318
|
+
const { span, startTime } = spanData;
|
|
19319
|
+
if (isAsyncIterable4(event.result)) {
|
|
19320
|
+
patchStreamIfNeeded(event.result, {
|
|
19321
|
+
onComplete: (chunks) => {
|
|
19322
|
+
try {
|
|
19323
|
+
const { output, metrics } = config.aggregateChunks(
|
|
19324
|
+
chunks,
|
|
19325
|
+
startTime
|
|
19326
|
+
);
|
|
19327
|
+
span.log({
|
|
19328
|
+
output,
|
|
19329
|
+
metrics
|
|
19330
|
+
});
|
|
19331
|
+
} catch (error) {
|
|
19332
|
+
console.error(
|
|
19333
|
+
`Error extracting output for ${channelName}:`,
|
|
19334
|
+
error
|
|
19335
|
+
);
|
|
19336
|
+
} finally {
|
|
19337
|
+
span.end();
|
|
19338
|
+
}
|
|
19339
|
+
},
|
|
19340
|
+
onError: (error) => {
|
|
19341
|
+
span.log({
|
|
19342
|
+
error: error.message
|
|
19343
|
+
});
|
|
19344
|
+
span.end();
|
|
19345
|
+
}
|
|
19346
|
+
});
|
|
19347
|
+
} else {
|
|
19348
|
+
span.end();
|
|
19349
|
+
spans.delete(event);
|
|
19350
|
+
}
|
|
19351
|
+
},
|
|
19352
|
+
error: (event) => {
|
|
19353
|
+
const spanData = spans.get(event);
|
|
19354
|
+
if (!spanData) {
|
|
19355
|
+
return;
|
|
19356
|
+
}
|
|
19357
|
+
const { span } = spanData;
|
|
19358
|
+
span.log({
|
|
19359
|
+
error: event.error.message
|
|
19360
|
+
});
|
|
19361
|
+
span.end();
|
|
19362
|
+
spans.delete(event);
|
|
19363
|
+
}
|
|
19364
|
+
};
|
|
19365
|
+
channel.subscribe(handlers);
|
|
19366
|
+
this.unsubscribers.push(() => {
|
|
19367
|
+
channel.unsubscribe(handlers);
|
|
19368
|
+
});
|
|
19369
|
+
}
|
|
19370
|
+
};
|
|
19371
|
+
function serializeInput2(params) {
|
|
19372
|
+
const input = {
|
|
19373
|
+
model: params.model,
|
|
19374
|
+
contents: serializeContents2(params.contents)
|
|
19375
|
+
};
|
|
19376
|
+
if (params.config) {
|
|
19377
|
+
const config = tryToDict2(params.config);
|
|
19378
|
+
if (config) {
|
|
19379
|
+
const tools = serializeTools2(params);
|
|
19380
|
+
if (tools) {
|
|
19381
|
+
config.tools = tools;
|
|
19382
|
+
}
|
|
19383
|
+
input.config = config;
|
|
19384
|
+
}
|
|
19385
|
+
}
|
|
19386
|
+
return input;
|
|
19387
|
+
}
|
|
19388
|
+
function serializeContents2(contents) {
|
|
19389
|
+
if (contents === null || contents === void 0) {
|
|
19390
|
+
return null;
|
|
19391
|
+
}
|
|
19392
|
+
if (Array.isArray(contents)) {
|
|
19393
|
+
return contents.map((item) => serializeContentItem2(item));
|
|
19394
|
+
}
|
|
19395
|
+
return serializeContentItem2(contents);
|
|
19396
|
+
}
|
|
19397
|
+
function serializeContentItem2(item) {
|
|
19398
|
+
if (typeof item === "object" && item !== null) {
|
|
19399
|
+
if (item.parts && Array.isArray(item.parts)) {
|
|
19400
|
+
return {
|
|
19401
|
+
...item,
|
|
19402
|
+
parts: item.parts.map((part) => serializePart2(part))
|
|
19403
|
+
};
|
|
19404
|
+
}
|
|
19405
|
+
return item;
|
|
19406
|
+
}
|
|
19407
|
+
if (typeof item === "string") {
|
|
19408
|
+
return { text: item };
|
|
19409
|
+
}
|
|
19410
|
+
return item;
|
|
19411
|
+
}
|
|
19412
|
+
function serializePart2(part) {
|
|
19413
|
+
if (!part || typeof part !== "object") {
|
|
19414
|
+
return part;
|
|
19415
|
+
}
|
|
19416
|
+
if (part.inlineData && part.inlineData.data) {
|
|
19417
|
+
const { data, mimeType } = part.inlineData;
|
|
19418
|
+
if (data instanceof Uint8Array || typeof Buffer !== "undefined" && Buffer.isBuffer(data) || typeof data === "string") {
|
|
19419
|
+
const extension = mimeType ? mimeType.split("/")[1] : "bin";
|
|
19420
|
+
const filename = `file.${extension}`;
|
|
19421
|
+
const buffer = typeof data === "string" ? typeof Buffer !== "undefined" ? Buffer.from(data, "base64") : new Uint8Array(
|
|
19422
|
+
atob(data).split("").map((c) => c.charCodeAt(0))
|
|
19423
|
+
) : typeof Buffer !== "undefined" ? Buffer.from(data) : new Uint8Array(data);
|
|
19424
|
+
const attachment = new Attachment({
|
|
19425
|
+
data: buffer,
|
|
19426
|
+
filename,
|
|
19427
|
+
contentType: mimeType || "application/octet-stream"
|
|
19428
|
+
});
|
|
19429
|
+
return {
|
|
19430
|
+
image_url: { url: attachment }
|
|
19431
|
+
};
|
|
19432
|
+
}
|
|
19433
|
+
}
|
|
19434
|
+
return part;
|
|
19435
|
+
}
|
|
19436
|
+
function serializeTools2(params) {
|
|
19437
|
+
if (!params.config?.tools) {
|
|
19438
|
+
return null;
|
|
19439
|
+
}
|
|
19440
|
+
try {
|
|
19441
|
+
return params.config.tools.map((tool) => {
|
|
19442
|
+
if (typeof tool === "object" && tool.functionDeclarations) {
|
|
19443
|
+
return tool;
|
|
19444
|
+
}
|
|
19445
|
+
return tool;
|
|
19446
|
+
});
|
|
19447
|
+
} catch {
|
|
19448
|
+
return null;
|
|
19449
|
+
}
|
|
19450
|
+
}
|
|
19451
|
+
function extractMetadata2(params) {
|
|
19452
|
+
const metadata = {};
|
|
19453
|
+
if (params.model) {
|
|
19454
|
+
metadata.model = params.model;
|
|
19455
|
+
}
|
|
19456
|
+
if (params.config) {
|
|
19457
|
+
const config = tryToDict2(params.config);
|
|
19458
|
+
if (config) {
|
|
19459
|
+
Object.keys(config).forEach((key) => {
|
|
19460
|
+
if (key !== "tools") {
|
|
19461
|
+
metadata[key] = config[key];
|
|
19462
|
+
}
|
|
19463
|
+
});
|
|
19464
|
+
}
|
|
19465
|
+
}
|
|
19466
|
+
return metadata;
|
|
19467
|
+
}
|
|
19468
|
+
function extractGenerateContentMetrics2(response, startTime) {
|
|
19469
|
+
const metrics = {};
|
|
19470
|
+
if (startTime) {
|
|
19471
|
+
const end = getCurrentUnixTimestamp();
|
|
19472
|
+
metrics.duration = end - startTime;
|
|
19473
|
+
}
|
|
19474
|
+
if (response.usageMetadata) {
|
|
19475
|
+
const usage = response.usageMetadata;
|
|
19476
|
+
if (usage.promptTokenCount !== void 0) {
|
|
19477
|
+
metrics.prompt_tokens = usage.promptTokenCount;
|
|
19478
|
+
}
|
|
19479
|
+
if (usage.candidatesTokenCount !== void 0) {
|
|
19480
|
+
metrics.completion_tokens = usage.candidatesTokenCount;
|
|
19481
|
+
}
|
|
19482
|
+
if (usage.totalTokenCount !== void 0) {
|
|
19483
|
+
metrics.tokens = usage.totalTokenCount;
|
|
19484
|
+
}
|
|
19485
|
+
if (usage.cachedContentTokenCount !== void 0) {
|
|
19486
|
+
metrics.prompt_cached_tokens = usage.cachedContentTokenCount;
|
|
19487
|
+
}
|
|
19488
|
+
if (usage.thoughtsTokenCount !== void 0) {
|
|
19489
|
+
metrics.completion_reasoning_tokens = usage.thoughtsTokenCount;
|
|
19490
|
+
}
|
|
19491
|
+
}
|
|
19492
|
+
return metrics;
|
|
19493
|
+
}
|
|
19494
|
+
function aggregateGenerateContentChunks2(chunks, startTime) {
|
|
19495
|
+
const end = getCurrentUnixTimestamp();
|
|
19496
|
+
const metrics = {
|
|
19497
|
+
duration: end - startTime
|
|
19498
|
+
};
|
|
19499
|
+
let firstTokenTime = null;
|
|
19500
|
+
if (chunks.length > 0 && firstTokenTime === null) {
|
|
19501
|
+
firstTokenTime = getCurrentUnixTimestamp();
|
|
19502
|
+
metrics.time_to_first_token = firstTokenTime - startTime;
|
|
19503
|
+
}
|
|
19504
|
+
if (chunks.length === 0) {
|
|
19505
|
+
return { output: {}, metrics };
|
|
19506
|
+
}
|
|
19507
|
+
let text = "";
|
|
19508
|
+
let thoughtText = "";
|
|
19509
|
+
const otherParts = [];
|
|
19510
|
+
let usageMetadata = null;
|
|
19511
|
+
let lastResponse = null;
|
|
19512
|
+
for (const chunk of chunks) {
|
|
19513
|
+
lastResponse = chunk;
|
|
19514
|
+
if (chunk.usageMetadata) {
|
|
19515
|
+
usageMetadata = chunk.usageMetadata;
|
|
19516
|
+
}
|
|
19517
|
+
if (chunk.candidates && Array.isArray(chunk.candidates)) {
|
|
19518
|
+
for (const candidate of chunk.candidates) {
|
|
19519
|
+
if (candidate.content?.parts) {
|
|
19520
|
+
for (const part of candidate.content.parts) {
|
|
19521
|
+
if (part.text !== void 0) {
|
|
19522
|
+
if (part.thought) {
|
|
19523
|
+
thoughtText += part.text;
|
|
19524
|
+
} else {
|
|
19525
|
+
text += part.text;
|
|
19526
|
+
}
|
|
19527
|
+
} else if (part.functionCall) {
|
|
19528
|
+
otherParts.push({ functionCall: part.functionCall });
|
|
19529
|
+
} else if (part.codeExecutionResult) {
|
|
19530
|
+
otherParts.push({
|
|
19531
|
+
codeExecutionResult: part.codeExecutionResult
|
|
19532
|
+
});
|
|
19533
|
+
} else if (part.executableCode) {
|
|
19534
|
+
otherParts.push({ executableCode: part.executableCode });
|
|
19535
|
+
}
|
|
19536
|
+
}
|
|
19537
|
+
}
|
|
19538
|
+
}
|
|
19539
|
+
}
|
|
19540
|
+
}
|
|
19541
|
+
const output = {};
|
|
19542
|
+
const parts = [];
|
|
19543
|
+
if (thoughtText) {
|
|
19544
|
+
parts.push({ text: thoughtText, thought: true });
|
|
19545
|
+
}
|
|
19546
|
+
if (text) {
|
|
19547
|
+
parts.push({ text });
|
|
19548
|
+
}
|
|
19549
|
+
parts.push(...otherParts);
|
|
19550
|
+
if (parts.length > 0 && lastResponse?.candidates) {
|
|
19551
|
+
const candidates = [];
|
|
19552
|
+
for (const candidate of lastResponse.candidates) {
|
|
19553
|
+
const candidateDict = {
|
|
19554
|
+
content: {
|
|
19555
|
+
parts,
|
|
19556
|
+
role: "model"
|
|
19557
|
+
}
|
|
19558
|
+
};
|
|
19559
|
+
if (candidate.finishReason !== void 0) {
|
|
19560
|
+
candidateDict.finishReason = candidate.finishReason;
|
|
19561
|
+
}
|
|
19562
|
+
if (candidate.safetyRatings) {
|
|
19563
|
+
candidateDict.safetyRatings = candidate.safetyRatings;
|
|
19564
|
+
}
|
|
19565
|
+
candidates.push(candidateDict);
|
|
19566
|
+
}
|
|
19567
|
+
output.candidates = candidates;
|
|
19568
|
+
}
|
|
19569
|
+
if (usageMetadata) {
|
|
19570
|
+
output.usageMetadata = usageMetadata;
|
|
19571
|
+
if (usageMetadata.promptTokenCount !== void 0) {
|
|
19572
|
+
metrics.prompt_tokens = usageMetadata.promptTokenCount;
|
|
19573
|
+
}
|
|
19574
|
+
if (usageMetadata.candidatesTokenCount !== void 0) {
|
|
19575
|
+
metrics.completion_tokens = usageMetadata.candidatesTokenCount;
|
|
19576
|
+
}
|
|
19577
|
+
if (usageMetadata.totalTokenCount !== void 0) {
|
|
19578
|
+
metrics.tokens = usageMetadata.totalTokenCount;
|
|
19579
|
+
}
|
|
19580
|
+
if (usageMetadata.cachedContentTokenCount !== void 0) {
|
|
19581
|
+
metrics.prompt_cached_tokens = usageMetadata.cachedContentTokenCount;
|
|
19582
|
+
}
|
|
19583
|
+
if (usageMetadata.thoughtsTokenCount !== void 0) {
|
|
19584
|
+
metrics.completion_reasoning_tokens = usageMetadata.thoughtsTokenCount;
|
|
19585
|
+
}
|
|
19586
|
+
}
|
|
19587
|
+
if (text) {
|
|
19588
|
+
output.text = text;
|
|
19589
|
+
}
|
|
19590
|
+
return { output, metrics };
|
|
19591
|
+
}
|
|
19592
|
+
function tryToDict2(obj) {
|
|
19593
|
+
if (obj === null || obj === void 0) {
|
|
19594
|
+
return null;
|
|
19595
|
+
}
|
|
19596
|
+
if (typeof obj === "object") {
|
|
19597
|
+
if (typeof obj.toJSON === "function") {
|
|
19598
|
+
return obj.toJSON();
|
|
19599
|
+
}
|
|
19600
|
+
return obj;
|
|
19601
|
+
}
|
|
19602
|
+
return null;
|
|
19603
|
+
}
|
|
19604
|
+
|
|
19605
|
+
// src/instrumentation/braintrust-plugin.ts
|
|
19606
|
+
var BraintrustPlugin = class extends BasePlugin {
|
|
19607
|
+
config;
|
|
19608
|
+
openaiPlugin = null;
|
|
19609
|
+
anthropicPlugin = null;
|
|
19610
|
+
aiSDKPlugin = null;
|
|
19611
|
+
claudeAgentSDKPlugin = null;
|
|
19612
|
+
googleGenAIPlugin = null;
|
|
19613
|
+
constructor(config = {}) {
|
|
19614
|
+
super();
|
|
19615
|
+
this.config = config;
|
|
19616
|
+
}
|
|
19617
|
+
onEnable() {
|
|
19618
|
+
const integrations = this.config.integrations || {};
|
|
19619
|
+
if (integrations.openai !== false) {
|
|
19620
|
+
this.openaiPlugin = new OpenAIPlugin();
|
|
19621
|
+
this.openaiPlugin.enable();
|
|
19622
|
+
}
|
|
19623
|
+
if (integrations.anthropic !== false) {
|
|
19624
|
+
this.anthropicPlugin = new AnthropicPlugin();
|
|
19625
|
+
this.anthropicPlugin.enable();
|
|
19626
|
+
}
|
|
19627
|
+
if (integrations.aisdk !== false && integrations.vercel !== false) {
|
|
19628
|
+
this.aiSDKPlugin = new AISDKPlugin();
|
|
19629
|
+
this.aiSDKPlugin.enable();
|
|
19630
|
+
}
|
|
19631
|
+
if (integrations.claudeAgentSDK !== false) {
|
|
19632
|
+
this.claudeAgentSDKPlugin = new ClaudeAgentSDKPlugin();
|
|
19633
|
+
this.claudeAgentSDKPlugin.enable();
|
|
19634
|
+
}
|
|
19635
|
+
if (integrations.googleGenAI !== false && integrations.google !== false) {
|
|
19636
|
+
this.googleGenAIPlugin = new GoogleGenAIPlugin();
|
|
19637
|
+
this.googleGenAIPlugin.enable();
|
|
19638
|
+
}
|
|
19639
|
+
}
|
|
19640
|
+
onDisable() {
|
|
19641
|
+
if (this.openaiPlugin) {
|
|
19642
|
+
this.openaiPlugin.disable();
|
|
19643
|
+
this.openaiPlugin = null;
|
|
19644
|
+
}
|
|
19645
|
+
if (this.anthropicPlugin) {
|
|
19646
|
+
this.anthropicPlugin.disable();
|
|
19647
|
+
this.anthropicPlugin = null;
|
|
19648
|
+
}
|
|
19649
|
+
if (this.aiSDKPlugin) {
|
|
19650
|
+
this.aiSDKPlugin.disable();
|
|
19651
|
+
this.aiSDKPlugin = null;
|
|
19652
|
+
}
|
|
19653
|
+
if (this.claudeAgentSDKPlugin) {
|
|
19654
|
+
this.claudeAgentSDKPlugin.disable();
|
|
19655
|
+
this.claudeAgentSDKPlugin = null;
|
|
19656
|
+
}
|
|
19657
|
+
if (this.googleGenAIPlugin) {
|
|
19658
|
+
this.googleGenAIPlugin.disable();
|
|
19659
|
+
this.googleGenAIPlugin = null;
|
|
19660
|
+
}
|
|
19661
|
+
}
|
|
19662
|
+
};
|
|
19663
|
+
|
|
19664
|
+
// src/instrumentation/registry.ts
|
|
19665
|
+
var PluginRegistry = class {
|
|
19666
|
+
braintrustPlugin = null;
|
|
19667
|
+
config = {};
|
|
19668
|
+
enabled = false;
|
|
19669
|
+
/**
|
|
19670
|
+
* Configure which integrations should be enabled.
|
|
19671
|
+
* This must be called before any SDK imports to take effect.
|
|
19672
|
+
*/
|
|
19673
|
+
configure(config) {
|
|
19674
|
+
if (this.enabled) {
|
|
19675
|
+
console.warn(
|
|
19676
|
+
"Braintrust: Cannot configure instrumentation after it has been enabled. Call configureInstrumentation() before importing any AI SDKs."
|
|
19677
|
+
);
|
|
19678
|
+
return;
|
|
19679
|
+
}
|
|
19680
|
+
this.config = { ...this.config, ...config };
|
|
19681
|
+
}
|
|
19682
|
+
/**
|
|
19683
|
+
* Enable all configured plugins.
|
|
19684
|
+
* Called automatically when the library is loaded.
|
|
19685
|
+
*/
|
|
19686
|
+
enable() {
|
|
19687
|
+
if (this.enabled) {
|
|
19688
|
+
return;
|
|
19689
|
+
}
|
|
19690
|
+
this.enabled = true;
|
|
19691
|
+
const envConfig = this.readEnvConfig();
|
|
19692
|
+
const finalConfig = {
|
|
19693
|
+
integrations: {
|
|
19694
|
+
...this.getDefaultConfig(),
|
|
19695
|
+
...this.config.integrations,
|
|
19696
|
+
...envConfig.integrations
|
|
19697
|
+
}
|
|
19698
|
+
};
|
|
19699
|
+
this.braintrustPlugin = new BraintrustPlugin(finalConfig);
|
|
19700
|
+
this.braintrustPlugin.enable();
|
|
19701
|
+
}
|
|
19702
|
+
/**
|
|
19703
|
+
* Disable all plugins.
|
|
19704
|
+
* Primarily used for testing.
|
|
19705
|
+
*/
|
|
19706
|
+
disable() {
|
|
19707
|
+
if (!this.enabled) {
|
|
19708
|
+
return;
|
|
19709
|
+
}
|
|
19710
|
+
this.enabled = false;
|
|
19711
|
+
if (this.braintrustPlugin) {
|
|
19712
|
+
this.braintrustPlugin.disable();
|
|
19713
|
+
this.braintrustPlugin = null;
|
|
19714
|
+
}
|
|
19715
|
+
}
|
|
19716
|
+
/**
|
|
19717
|
+
* Check if instrumentation is enabled.
|
|
19718
|
+
*/
|
|
19719
|
+
isEnabled() {
|
|
19720
|
+
return this.enabled;
|
|
19721
|
+
}
|
|
19722
|
+
/**
|
|
19723
|
+
* Get default configuration (all integrations enabled).
|
|
19724
|
+
*/
|
|
19725
|
+
getDefaultConfig() {
|
|
19726
|
+
return {
|
|
19727
|
+
openai: true,
|
|
19728
|
+
anthropic: true,
|
|
19729
|
+
vercel: true,
|
|
19730
|
+
aisdk: true,
|
|
19731
|
+
google: true,
|
|
19732
|
+
claudeAgentSDK: true
|
|
19733
|
+
};
|
|
19734
|
+
}
|
|
19735
|
+
/**
|
|
19736
|
+
* Read configuration from environment variables.
|
|
19737
|
+
* Supports: BRAINTRUST_DISABLE_INSTRUMENTATION=openai,anthropic,...
|
|
19738
|
+
*/
|
|
19739
|
+
readEnvConfig() {
|
|
19740
|
+
const integrations = {};
|
|
19741
|
+
const disabledList = isomorph_default.getEnv("BRAINTRUST_DISABLE_INSTRUMENTATION");
|
|
19742
|
+
if (disabledList) {
|
|
19743
|
+
const disabled = disabledList.split(",").map((s) => s.trim().toLowerCase()).filter((s) => s.length > 0);
|
|
19744
|
+
for (const sdk of disabled) {
|
|
19745
|
+
integrations[sdk] = false;
|
|
19746
|
+
}
|
|
19747
|
+
}
|
|
19748
|
+
return { integrations };
|
|
19749
|
+
}
|
|
19750
|
+
};
|
|
19751
|
+
var registry = new PluginRegistry();
|
|
19752
|
+
function configureInstrumentation(config) {
|
|
19753
|
+
registry.configure(config);
|
|
19754
|
+
}
|
|
19755
|
+
|
|
16710
19756
|
// src/workerd/index.ts
|
|
16711
19757
|
configureWorkerd();
|
|
16712
19758
|
export {
|
|
@@ -16762,6 +19808,7 @@ export {
|
|
|
16762
19808
|
addAzureBlobHeaders,
|
|
16763
19809
|
braintrustStreamChunkSchema,
|
|
16764
19810
|
buildLocalSummary,
|
|
19811
|
+
configureInstrumentation,
|
|
16765
19812
|
constructLogs3OverflowRequest,
|
|
16766
19813
|
createFinalValuePassThroughStream,
|
|
16767
19814
|
currentExperiment,
|
|
@@ -16838,5 +19885,6 @@ export {
|
|
|
16838
19885
|
wrapMastraAgent,
|
|
16839
19886
|
wrapOpenAI,
|
|
16840
19887
|
wrapOpenAIv4,
|
|
16841
|
-
wrapTraced
|
|
19888
|
+
wrapTraced,
|
|
19889
|
+
wrapVitest
|
|
16842
19890
|
};
|