@librechat/agents 3.1.96 → 3.1.98
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/graphs/Graph.cjs +60 -21
- package/dist/cjs/graphs/Graph.cjs.map +1 -1
- package/dist/cjs/instrumentation.cjs +120 -9
- package/dist/cjs/instrumentation.cjs.map +1 -1
- package/dist/cjs/langfuse.cjs +30 -226
- package/dist/cjs/langfuse.cjs.map +1 -1
- package/dist/cjs/langfuseToolOutputTracing.cjs +476 -0
- package/dist/cjs/langfuseToolOutputTracing.cjs.map +1 -0
- package/dist/cjs/llm/bedrock/index.cjs +10 -0
- package/dist/cjs/llm/bedrock/index.cjs.map +1 -1
- package/dist/cjs/llm/bedrock/toolCache.cjs +125 -0
- package/dist/cjs/llm/bedrock/toolCache.cjs.map +1 -0
- package/dist/cjs/messages/cache.cjs +17 -9
- package/dist/cjs/messages/cache.cjs.map +1 -1
- package/dist/cjs/run.cjs +142 -69
- package/dist/cjs/run.cjs.map +1 -1
- package/dist/cjs/tools/ToolNode.cjs +26 -9
- package/dist/cjs/tools/ToolNode.cjs.map +1 -1
- package/dist/cjs/tools/subagent/SubagentExecutor.cjs +10 -6
- package/dist/cjs/tools/subagent/SubagentExecutor.cjs.map +1 -1
- package/dist/esm/graphs/Graph.mjs +62 -23
- package/dist/esm/graphs/Graph.mjs.map +1 -1
- package/dist/esm/instrumentation.mjs +118 -9
- package/dist/esm/instrumentation.mjs.map +1 -1
- package/dist/esm/langfuse.mjs +28 -224
- package/dist/esm/langfuse.mjs.map +1 -1
- package/dist/esm/langfuseToolOutputTracing.mjs +468 -0
- package/dist/esm/langfuseToolOutputTracing.mjs.map +1 -0
- package/dist/esm/llm/bedrock/index.mjs +10 -0
- package/dist/esm/llm/bedrock/index.mjs.map +1 -1
- package/dist/esm/llm/bedrock/toolCache.mjs +122 -0
- package/dist/esm/llm/bedrock/toolCache.mjs.map +1 -0
- package/dist/esm/messages/cache.mjs +17 -9
- package/dist/esm/messages/cache.mjs.map +1 -1
- package/dist/esm/run.mjs +144 -71
- package/dist/esm/run.mjs.map +1 -1
- package/dist/esm/tools/ToolNode.mjs +26 -9
- package/dist/esm/tools/ToolNode.mjs.map +1 -1
- package/dist/esm/tools/subagent/SubagentExecutor.mjs +10 -6
- package/dist/esm/tools/subagent/SubagentExecutor.mjs.map +1 -1
- package/dist/types/graphs/Graph.d.ts +5 -1
- package/dist/types/instrumentation.d.ts +5 -1
- package/dist/types/langfuse.d.ts +6 -28
- package/dist/types/langfuseToolOutputTracing.d.ts +20 -0
- package/dist/types/llm/bedrock/index.d.ts +16 -0
- package/dist/types/llm/bedrock/toolCache.d.ts +4 -0
- package/dist/types/messages/cache.d.ts +2 -2
- package/dist/types/run.d.ts +5 -1
- package/dist/types/tools/ToolNode.d.ts +4 -1
- package/dist/types/tools/subagent/SubagentExecutor.d.ts +2 -0
- package/dist/types/types/graph.d.ts +30 -0
- package/dist/types/types/llm.d.ts +2 -2
- package/dist/types/types/run.d.ts +6 -0
- package/dist/types/types/tools.d.ts +7 -0
- package/package.json +2 -1
- package/src/agents/__tests__/AgentContext.anthropic.live.test.ts +332 -0
- package/src/agents/__tests__/AgentContext.bedrock.live.test.ts +504 -0
- package/src/graphs/Graph.ts +104 -34
- package/src/instrumentation.ts +172 -11
- package/src/langfuse.ts +59 -324
- package/src/langfuseToolOutputTracing.ts +702 -0
- package/src/llm/bedrock/index.ts +32 -1
- package/src/llm/bedrock/llm.spec.ts +154 -1
- package/src/llm/bedrock/toolCache.test.ts +131 -0
- package/src/llm/bedrock/toolCache.ts +191 -0
- package/src/messages/cache.test.ts +97 -38
- package/src/messages/cache.ts +18 -10
- package/src/run.ts +190 -87
- package/src/specs/langfuse-callbacks.test.ts +178 -1
- package/src/specs/langfuse-config.test.ts +112 -76
- package/src/specs/langfuse-instrumentation.test.ts +283 -0
- package/src/specs/langfuse-metadata.test.ts +54 -1
- package/src/specs/langfuse-tool-output-tracing.test.ts +616 -0
- package/src/tools/ToolNode.ts +35 -8
- package/src/tools/__tests__/SubagentExecutor.test.ts +32 -0
- package/src/tools/__tests__/ToolNode.langfuse.test.ts +47 -0
- package/src/tools/subagent/SubagentExecutor.ts +11 -6
- package/src/types/graph.ts +32 -0
- package/src/types/llm.ts +2 -2
- package/src/types/run.ts +6 -0
- package/src/types/tools.ts +7 -0
|
@@ -0,0 +1,616 @@
|
|
|
1
|
+
import { AIMessage, ToolMessage, HumanMessage } from '@langchain/core/messages';
|
|
2
|
+
import { LangfuseOtelSpanAttributes } from '@langfuse/tracing';
|
|
3
|
+
import type { ReadableSpan } from '@opentelemetry/sdk-trace-base';
|
|
4
|
+
import type { BaseMessage } from '@langchain/core/messages';
|
|
5
|
+
import type { TPayload } from '@/types';
|
|
6
|
+
import {
|
|
7
|
+
LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT,
|
|
8
|
+
redactLangfuseSpanToolOutputs,
|
|
9
|
+
resolveLangfuseConfig,
|
|
10
|
+
shouldTraceToolNodeForLangfuse,
|
|
11
|
+
type ResolvedLangfuseToolOutputTracingConfig,
|
|
12
|
+
} from '@/langfuseToolOutputTracing';
|
|
13
|
+
import { formatAgentMessages } from '@/messages/format';
|
|
14
|
+
import { ContentTypes } from '@/common';
|
|
15
|
+
|
|
16
|
+
type SerializedLangfuseChatMessage = {
|
|
17
|
+
content: BaseMessage['content'];
|
|
18
|
+
role?: string;
|
|
19
|
+
additional_kwargs?: BaseMessage['additional_kwargs'];
|
|
20
|
+
tool_calls?:
|
|
21
|
+
| NonNullable<AIMessage['tool_calls']>
|
|
22
|
+
| NonNullable<BaseMessage['additional_kwargs']['tool_calls']>;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
type RedactedMessage = {
|
|
26
|
+
role?: string;
|
|
27
|
+
content?: string;
|
|
28
|
+
tool_calls?: Array<{
|
|
29
|
+
id?: string;
|
|
30
|
+
name?: string;
|
|
31
|
+
args?: {
|
|
32
|
+
query?: string;
|
|
33
|
+
};
|
|
34
|
+
}>;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
function createSpan(
|
|
38
|
+
name: string,
|
|
39
|
+
attributes: Record<string, unknown>
|
|
40
|
+
): ReadableSpan {
|
|
41
|
+
return { name, attributes } as unknown as ReadableSpan;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function createConfig(
|
|
45
|
+
overrides: Partial<ResolvedLangfuseToolOutputTracingConfig> = {}
|
|
46
|
+
): ResolvedLangfuseToolOutputTracingConfig {
|
|
47
|
+
return {
|
|
48
|
+
enabled: true,
|
|
49
|
+
redactedToolNames: new Set<string>(),
|
|
50
|
+
redactedToolNameMatchMode: 'exact',
|
|
51
|
+
redactionText: LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT,
|
|
52
|
+
...overrides,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function serializeMessageForLangfuse(
|
|
57
|
+
message: BaseMessage
|
|
58
|
+
): SerializedLangfuseChatMessage {
|
|
59
|
+
if (message instanceof HumanMessage) {
|
|
60
|
+
return { content: message.content, role: 'user' };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (message instanceof AIMessage) {
|
|
64
|
+
const response: SerializedLangfuseChatMessage = {
|
|
65
|
+
content: message.content,
|
|
66
|
+
role: 'assistant',
|
|
67
|
+
};
|
|
68
|
+
if (message.tool_calls != null && message.tool_calls.length > 0) {
|
|
69
|
+
response.tool_calls = message.tool_calls;
|
|
70
|
+
}
|
|
71
|
+
if (message.additional_kwargs.tool_calls != null) {
|
|
72
|
+
response.tool_calls = message.additional_kwargs.tool_calls;
|
|
73
|
+
}
|
|
74
|
+
return response;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (message instanceof ToolMessage) {
|
|
78
|
+
return {
|
|
79
|
+
content: message.content,
|
|
80
|
+
additional_kwargs: message.additional_kwargs,
|
|
81
|
+
role: message.name,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return message.name != null
|
|
86
|
+
? { content: message.content, role: message.name }
|
|
87
|
+
: { content: message.content };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function readJsonAttribute<T>(span: ReadableSpan, key: string): T {
|
|
91
|
+
return JSON.parse(span.attributes[key] as string) as T;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
describe('Langfuse tool output tracing redaction', () => {
|
|
95
|
+
const originalEnv = process.env;
|
|
96
|
+
|
|
97
|
+
beforeEach(() => {
|
|
98
|
+
process.env = { ...originalEnv };
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
afterEach(() => {
|
|
102
|
+
process.env = originalEnv;
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it('enables ToolNode tracing only when Langfuse is active by default', () => {
|
|
106
|
+
delete process.env.LANGFUSE_SECRET_KEY;
|
|
107
|
+
delete process.env.LANGFUSE_PUBLIC_KEY;
|
|
108
|
+
delete process.env.LANGFUSE_BASE_URL;
|
|
109
|
+
|
|
110
|
+
expect(shouldTraceToolNodeForLangfuse({})).toBe(false);
|
|
111
|
+
expect(
|
|
112
|
+
shouldTraceToolNodeForLangfuse({
|
|
113
|
+
runLangfuse: {
|
|
114
|
+
enabled: true,
|
|
115
|
+
publicKey: 'pk-run',
|
|
116
|
+
secretKey: 'sk-run',
|
|
117
|
+
},
|
|
118
|
+
})
|
|
119
|
+
).toBe(true);
|
|
120
|
+
expect(
|
|
121
|
+
shouldTraceToolNodeForLangfuse({
|
|
122
|
+
agentLangfuse: {
|
|
123
|
+
enabled: true,
|
|
124
|
+
publicKey: 'pk-agent',
|
|
125
|
+
secretKey: 'sk-agent',
|
|
126
|
+
baseUrl: 'https://langfuse.test',
|
|
127
|
+
toolNodeTracing: { enabled: true },
|
|
128
|
+
},
|
|
129
|
+
})
|
|
130
|
+
).toBe(true);
|
|
131
|
+
|
|
132
|
+
process.env.LANGFUSE_SECRET_KEY = 'sk-test';
|
|
133
|
+
process.env.LANGFUSE_PUBLIC_KEY = 'pk-test';
|
|
134
|
+
process.env.LANGFUSE_BASE_URL = 'https://langfuse.test';
|
|
135
|
+
|
|
136
|
+
expect(shouldTraceToolNodeForLangfuse({})).toBe(true);
|
|
137
|
+
expect(
|
|
138
|
+
shouldTraceToolNodeForLangfuse({
|
|
139
|
+
runLangfuse: { toolNodeTracing: { enabled: true } },
|
|
140
|
+
})
|
|
141
|
+
).toBe(true);
|
|
142
|
+
expect(
|
|
143
|
+
shouldTraceToolNodeForLangfuse({
|
|
144
|
+
runLangfuse: { toolNodeTracing: { enabled: false } },
|
|
145
|
+
})
|
|
146
|
+
).toBe(false);
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it('lets agent Langfuse enablement override disabled run defaults for ToolNode tracing', () => {
|
|
150
|
+
delete process.env.LANGFUSE_SECRET_KEY;
|
|
151
|
+
delete process.env.LANGFUSE_PUBLIC_KEY;
|
|
152
|
+
delete process.env.LANGFUSE_BASE_URL;
|
|
153
|
+
|
|
154
|
+
expect(
|
|
155
|
+
shouldTraceToolNodeForLangfuse({
|
|
156
|
+
runLangfuse: {
|
|
157
|
+
enabled: false,
|
|
158
|
+
},
|
|
159
|
+
agentLangfuse: {
|
|
160
|
+
enabled: true,
|
|
161
|
+
publicKey: 'pk-agent',
|
|
162
|
+
secretKey: 'sk-agent',
|
|
163
|
+
baseUrl: 'https://langfuse.test',
|
|
164
|
+
},
|
|
165
|
+
})
|
|
166
|
+
).toBe(true);
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
it('keeps ToolNode tracing disabled when resolved Langfuse is disabled', () => {
|
|
170
|
+
process.env.LANGFUSE_SECRET_KEY = 'sk-test';
|
|
171
|
+
process.env.LANGFUSE_PUBLIC_KEY = 'pk-test';
|
|
172
|
+
|
|
173
|
+
expect(
|
|
174
|
+
shouldTraceToolNodeForLangfuse({
|
|
175
|
+
runLangfuse: {
|
|
176
|
+
enabled: false,
|
|
177
|
+
toolNodeTracing: { enabled: true },
|
|
178
|
+
},
|
|
179
|
+
})
|
|
180
|
+
).toBe(false);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it('classifies LangGraph tool-node spans as Langfuse tool observations', () => {
|
|
184
|
+
const span = createSpan('tool_batch', {
|
|
185
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: 'span',
|
|
186
|
+
[`${LangfuseOtelSpanAttributes.OBSERVATION_METADATA}.langgraph_node`]:
|
|
187
|
+
'tools=agent_1',
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
redactLangfuseSpanToolOutputs(span, createConfig());
|
|
191
|
+
|
|
192
|
+
expect(span.attributes[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]).toBe(
|
|
193
|
+
'tool'
|
|
194
|
+
);
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
it('does not reclassify non-tool LangGraph spans', () => {
|
|
198
|
+
const span = createSpan('agent=agent_1', {
|
|
199
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: 'span',
|
|
200
|
+
[`${LangfuseOtelSpanAttributes.OBSERVATION_METADATA}.langgraph_node`]:
|
|
201
|
+
'agent=agent_1',
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
redactLangfuseSpanToolOutputs(span, createConfig());
|
|
205
|
+
|
|
206
|
+
expect(span.attributes[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]).toBe(
|
|
207
|
+
'span'
|
|
208
|
+
);
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
it('redacts raw tool observation output when tool output tracing is disabled', () => {
|
|
212
|
+
const span = createSpan('execute_sql', {
|
|
213
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: 'tool',
|
|
214
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: '{"query":"select 1"}',
|
|
215
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]: 'secret rows',
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
redactLangfuseSpanToolOutputs(span, createConfig({ enabled: false }));
|
|
219
|
+
|
|
220
|
+
expect(span.attributes[LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]).toBe(
|
|
221
|
+
LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT
|
|
222
|
+
);
|
|
223
|
+
expect(span.attributes[LangfuseOtelSpanAttributes.OBSERVATION_INPUT]).toBe(
|
|
224
|
+
'{"query":"select 1"}'
|
|
225
|
+
);
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
it('redacts ToolMessage content inside serialized generation inputs', () => {
|
|
229
|
+
const messages = [
|
|
230
|
+
{ role: 'user', content: 'show tables' },
|
|
231
|
+
{
|
|
232
|
+
role: 'execute_sql',
|
|
233
|
+
content: 'private query result',
|
|
234
|
+
additional_kwargs: {},
|
|
235
|
+
},
|
|
236
|
+
];
|
|
237
|
+
const span = createSpan('gpt-4o', {
|
|
238
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: 'generation',
|
|
239
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: JSON.stringify(messages),
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
redactLangfuseSpanToolOutputs(span, createConfig({ enabled: false }));
|
|
243
|
+
|
|
244
|
+
const redacted = JSON.parse(
|
|
245
|
+
span.attributes[LangfuseOtelSpanAttributes.OBSERVATION_INPUT] as string
|
|
246
|
+
) as Array<{ role: string; content: string }>;
|
|
247
|
+
expect(redacted[0].content).toBe('show tables');
|
|
248
|
+
expect(redacted[1].content).toBe(LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT);
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
it('redacts only configured tool names when output tracing stays enabled', () => {
|
|
252
|
+
const messages = [
|
|
253
|
+
{ role: 'execute_sql', content: 'private query result' },
|
|
254
|
+
{ role: 'bash', content: 'public build log' },
|
|
255
|
+
];
|
|
256
|
+
const span = createSpan('LangGraph', {
|
|
257
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]: JSON.stringify({
|
|
258
|
+
messages,
|
|
259
|
+
}),
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
redactLangfuseSpanToolOutputs(
|
|
263
|
+
span,
|
|
264
|
+
createConfig({
|
|
265
|
+
redactedToolNames: new Set(['execute_sql']),
|
|
266
|
+
})
|
|
267
|
+
);
|
|
268
|
+
|
|
269
|
+
const redacted = JSON.parse(
|
|
270
|
+
span.attributes[LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT] as string
|
|
271
|
+
) as { messages: Array<{ role: string; content: string }> };
|
|
272
|
+
expect(redacted.messages[0].content).toBe(
|
|
273
|
+
LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT
|
|
274
|
+
);
|
|
275
|
+
expect(redacted.messages[1].content).toBe('public build log');
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
it('uses nested ToolMessage names instead of generic tool role', () => {
|
|
279
|
+
const messages = [
|
|
280
|
+
{
|
|
281
|
+
role: 'tool',
|
|
282
|
+
content: 'private query result',
|
|
283
|
+
kwargs: {
|
|
284
|
+
name: 'execute_sql',
|
|
285
|
+
tool_call_id: 'call_1',
|
|
286
|
+
},
|
|
287
|
+
},
|
|
288
|
+
];
|
|
289
|
+
const span = createSpan('gpt-4o', {
|
|
290
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: 'generation',
|
|
291
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: JSON.stringify(messages),
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
redactLangfuseSpanToolOutputs(
|
|
295
|
+
span,
|
|
296
|
+
createConfig({
|
|
297
|
+
redactedToolNames: new Set(['execute_sql']),
|
|
298
|
+
})
|
|
299
|
+
);
|
|
300
|
+
|
|
301
|
+
const redacted = readJsonAttribute<Array<{ content: string }>>(
|
|
302
|
+
span,
|
|
303
|
+
LangfuseOtelSpanAttributes.OBSERVATION_INPUT
|
|
304
|
+
);
|
|
305
|
+
expect(redacted[0].content).toBe(LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT);
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
it('maps tool_call_id to the preceding tool call name for allowlisted redaction', () => {
|
|
309
|
+
const messages = [
|
|
310
|
+
{
|
|
311
|
+
role: 'assistant',
|
|
312
|
+
content: '',
|
|
313
|
+
tool_calls: [
|
|
314
|
+
{
|
|
315
|
+
id: 'call_sql',
|
|
316
|
+
name: 'execute_sql',
|
|
317
|
+
args: { query: 'select * from private_table' },
|
|
318
|
+
},
|
|
319
|
+
],
|
|
320
|
+
},
|
|
321
|
+
{
|
|
322
|
+
role: 'tool',
|
|
323
|
+
tool_call_id: 'call_sql',
|
|
324
|
+
content: 'sensitive row output',
|
|
325
|
+
},
|
|
326
|
+
{
|
|
327
|
+
role: 'tool',
|
|
328
|
+
tool_call_id: 'call_bash',
|
|
329
|
+
content: 'public build log',
|
|
330
|
+
},
|
|
331
|
+
];
|
|
332
|
+
const span = createSpan('gpt-4o', {
|
|
333
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: 'generation',
|
|
334
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: JSON.stringify(messages),
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
redactLangfuseSpanToolOutputs(
|
|
338
|
+
span,
|
|
339
|
+
createConfig({
|
|
340
|
+
redactedToolNames: new Set(['execute_sql']),
|
|
341
|
+
})
|
|
342
|
+
);
|
|
343
|
+
|
|
344
|
+
const redacted = readJsonAttribute<RedactedMessage[]>(
|
|
345
|
+
span,
|
|
346
|
+
LangfuseOtelSpanAttributes.OBSERVATION_INPUT
|
|
347
|
+
);
|
|
348
|
+
expect(redacted[0].tool_calls?.[0]?.args?.query).toBe(
|
|
349
|
+
'select * from private_table'
|
|
350
|
+
);
|
|
351
|
+
expect(redacted[1].content).toBe(LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT);
|
|
352
|
+
expect(redacted[2].content).toBe('public build log');
|
|
353
|
+
});
|
|
354
|
+
|
|
355
|
+
it('does not redact partial tool name matches by default', () => {
|
|
356
|
+
const span = createSpan('clickhouse_execute_sql_prod', {
|
|
357
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: 'tool',
|
|
358
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]: 'secret rows',
|
|
359
|
+
});
|
|
360
|
+
|
|
361
|
+
redactLangfuseSpanToolOutputs(
|
|
362
|
+
span,
|
|
363
|
+
createConfig({
|
|
364
|
+
redactedToolNames: new Set(['execute_sql']),
|
|
365
|
+
})
|
|
366
|
+
);
|
|
367
|
+
|
|
368
|
+
expect(span.attributes[LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]).toBe(
|
|
369
|
+
'secret rows'
|
|
370
|
+
);
|
|
371
|
+
});
|
|
372
|
+
|
|
373
|
+
it('redacts configured partial tool name matches when enabled', () => {
|
|
374
|
+
const span = createSpan('clickhouse_execute_sql_prod', {
|
|
375
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: 'tool',
|
|
376
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]: 'secret rows',
|
|
377
|
+
});
|
|
378
|
+
|
|
379
|
+
redactLangfuseSpanToolOutputs(
|
|
380
|
+
span,
|
|
381
|
+
createConfig({
|
|
382
|
+
redactedToolNames: new Set(['execute_sql']),
|
|
383
|
+
redactedToolNameMatchMode: 'partial',
|
|
384
|
+
})
|
|
385
|
+
);
|
|
386
|
+
|
|
387
|
+
expect(span.attributes[LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]).toBe(
|
|
388
|
+
LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT
|
|
389
|
+
);
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
it('redacts prior tool outputs from multi-turn generation inputs', () => {
|
|
393
|
+
const messages = [
|
|
394
|
+
{ role: 'user', content: 'run the query' },
|
|
395
|
+
{
|
|
396
|
+
role: 'assistant',
|
|
397
|
+
content: '',
|
|
398
|
+
tool_calls: [
|
|
399
|
+
{
|
|
400
|
+
id: 'call_sql',
|
|
401
|
+
name: 'execute_sql',
|
|
402
|
+
args: { query: 'select * from private_table' },
|
|
403
|
+
},
|
|
404
|
+
],
|
|
405
|
+
},
|
|
406
|
+
{
|
|
407
|
+
role: 'execute_sql',
|
|
408
|
+
content: 'sensitive row output',
|
|
409
|
+
additional_kwargs: {},
|
|
410
|
+
},
|
|
411
|
+
{ role: 'assistant', content: 'I found the answer.' },
|
|
412
|
+
{ role: 'user', content: 'explain the first row' },
|
|
413
|
+
];
|
|
414
|
+
const span = createSpan('gpt-4o', {
|
|
415
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: 'generation',
|
|
416
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: JSON.stringify(messages),
|
|
417
|
+
});
|
|
418
|
+
|
|
419
|
+
redactLangfuseSpanToolOutputs(span, createConfig({ enabled: false }));
|
|
420
|
+
|
|
421
|
+
const redacted = readJsonAttribute<RedactedMessage[]>(
|
|
422
|
+
span,
|
|
423
|
+
LangfuseOtelSpanAttributes.OBSERVATION_INPUT
|
|
424
|
+
);
|
|
425
|
+
expect(redacted[0].content).toBe('run the query');
|
|
426
|
+
expect(redacted[1].tool_calls?.[0]?.args?.query).toBe(
|
|
427
|
+
'select * from private_table'
|
|
428
|
+
);
|
|
429
|
+
expect(redacted[2].content).toBe(LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT);
|
|
430
|
+
expect(redacted[3].content).toBe('I found the answer.');
|
|
431
|
+
expect(redacted[4].content).toBe('explain the first row');
|
|
432
|
+
});
|
|
433
|
+
|
|
434
|
+
it('redacts tool outputs after formatAgentMessages rehydrates content parts', () => {
|
|
435
|
+
const payload: TPayload = [
|
|
436
|
+
{ role: 'user', content: 'show me the private numbers' },
|
|
437
|
+
{
|
|
438
|
+
role: 'assistant',
|
|
439
|
+
content: [
|
|
440
|
+
{
|
|
441
|
+
type: ContentTypes.TEXT,
|
|
442
|
+
[ContentTypes.TEXT]: 'I will query ClickHouse.',
|
|
443
|
+
tool_call_ids: ['call_sql'],
|
|
444
|
+
},
|
|
445
|
+
{
|
|
446
|
+
type: ContentTypes.TOOL_CALL,
|
|
447
|
+
tool_call: {
|
|
448
|
+
id: 'call_sql',
|
|
449
|
+
name: 'execute_sql',
|
|
450
|
+
args: '{"query":"select secret_value from prod"}',
|
|
451
|
+
output: 'secret_value: 12345',
|
|
452
|
+
},
|
|
453
|
+
},
|
|
454
|
+
],
|
|
455
|
+
},
|
|
456
|
+
{ role: 'user', content: 'can you summarize it?' },
|
|
457
|
+
];
|
|
458
|
+
const { messages } = formatAgentMessages(
|
|
459
|
+
payload,
|
|
460
|
+
undefined,
|
|
461
|
+
new Set(['execute_sql'])
|
|
462
|
+
);
|
|
463
|
+
const serialized = messages.map(serializeMessageForLangfuse);
|
|
464
|
+
const span = createSpan('gpt-4o', {
|
|
465
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: 'generation',
|
|
466
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_INPUT]:
|
|
467
|
+
JSON.stringify(serialized),
|
|
468
|
+
});
|
|
469
|
+
|
|
470
|
+
redactLangfuseSpanToolOutputs(
|
|
471
|
+
span,
|
|
472
|
+
createConfig({
|
|
473
|
+
redactedToolNames: new Set(['execute_sql']),
|
|
474
|
+
})
|
|
475
|
+
);
|
|
476
|
+
|
|
477
|
+
const redacted = readJsonAttribute<RedactedMessage[]>(
|
|
478
|
+
span,
|
|
479
|
+
LangfuseOtelSpanAttributes.OBSERVATION_INPUT
|
|
480
|
+
);
|
|
481
|
+
expect(redacted[1].tool_calls?.[0]?.args?.query).toBe(
|
|
482
|
+
'select secret_value from prod'
|
|
483
|
+
);
|
|
484
|
+
expect(redacted[2].role).toBe('execute_sql');
|
|
485
|
+
expect(redacted[2].content).toBe(LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT);
|
|
486
|
+
expect(JSON.stringify(redacted)).not.toContain('secret_value: 12345');
|
|
487
|
+
});
|
|
488
|
+
|
|
489
|
+
it('redacts constructor-serialized ToolMessages from rehydrated content parts', () => {
|
|
490
|
+
const payload: TPayload = [
|
|
491
|
+
{ role: 'user', content: 'show the stored result' },
|
|
492
|
+
{
|
|
493
|
+
role: 'assistant',
|
|
494
|
+
content: [
|
|
495
|
+
{
|
|
496
|
+
type: ContentTypes.TEXT,
|
|
497
|
+
[ContentTypes.TEXT]: 'I will query ClickHouse.',
|
|
498
|
+
tool_call_ids: ['call_sql'],
|
|
499
|
+
},
|
|
500
|
+
{
|
|
501
|
+
type: ContentTypes.TOOL_CALL,
|
|
502
|
+
tool_call: {
|
|
503
|
+
id: 'call_sql',
|
|
504
|
+
name: 'execute_sql',
|
|
505
|
+
args: '{"query":"select constructor_path from prod"}',
|
|
506
|
+
output: 'constructor path secret',
|
|
507
|
+
},
|
|
508
|
+
},
|
|
509
|
+
],
|
|
510
|
+
},
|
|
511
|
+
];
|
|
512
|
+
const { messages } = formatAgentMessages(
|
|
513
|
+
payload,
|
|
514
|
+
undefined,
|
|
515
|
+
new Set(['execute_sql'])
|
|
516
|
+
);
|
|
517
|
+
const span = createSpan('gpt-4o', {
|
|
518
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: 'generation',
|
|
519
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: JSON.stringify([
|
|
520
|
+
messages,
|
|
521
|
+
]),
|
|
522
|
+
});
|
|
523
|
+
|
|
524
|
+
redactLangfuseSpanToolOutputs(
|
|
525
|
+
span,
|
|
526
|
+
createConfig({
|
|
527
|
+
redactedToolNames: new Set(['execute_sql']),
|
|
528
|
+
})
|
|
529
|
+
);
|
|
530
|
+
|
|
531
|
+
const redacted = span.attributes[
|
|
532
|
+
LangfuseOtelSpanAttributes.OBSERVATION_INPUT
|
|
533
|
+
] as string;
|
|
534
|
+
expect(redacted).toContain('select constructor_path from prod');
|
|
535
|
+
expect(redacted).toContain(LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT);
|
|
536
|
+
expect(redacted).not.toContain('constructor path secret');
|
|
537
|
+
});
|
|
538
|
+
|
|
539
|
+
it('redacts ToolMessage artifacts because they are tool output', () => {
|
|
540
|
+
const messages = [
|
|
541
|
+
{
|
|
542
|
+
id: ['langchain_core', 'messages', 'ToolMessage'],
|
|
543
|
+
kwargs: {
|
|
544
|
+
name: 'execute_sql',
|
|
545
|
+
tool_call_id: 'call_sql',
|
|
546
|
+
content: 'safe display content',
|
|
547
|
+
artifact: {
|
|
548
|
+
rows: ['artifact secret row'],
|
|
549
|
+
},
|
|
550
|
+
},
|
|
551
|
+
},
|
|
552
|
+
];
|
|
553
|
+
const span = createSpan('gpt-4o', {
|
|
554
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: 'generation',
|
|
555
|
+
[LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: JSON.stringify(messages),
|
|
556
|
+
});
|
|
557
|
+
|
|
558
|
+
redactLangfuseSpanToolOutputs(
|
|
559
|
+
span,
|
|
560
|
+
createConfig({
|
|
561
|
+
redactedToolNames: new Set(['execute_sql']),
|
|
562
|
+
})
|
|
563
|
+
);
|
|
564
|
+
|
|
565
|
+
const redacted = readJsonAttribute<
|
|
566
|
+
Array<{
|
|
567
|
+
kwargs: {
|
|
568
|
+
artifact: string;
|
|
569
|
+
content: string;
|
|
570
|
+
};
|
|
571
|
+
}>
|
|
572
|
+
>(span, LangfuseOtelSpanAttributes.OBSERVATION_INPUT);
|
|
573
|
+
expect(redacted[0].kwargs.content).toBe(
|
|
574
|
+
LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT
|
|
575
|
+
);
|
|
576
|
+
expect(redacted[0].kwargs.artifact).toBe(
|
|
577
|
+
LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT
|
|
578
|
+
);
|
|
579
|
+
expect(JSON.stringify(redacted)).not.toContain('artifact secret row');
|
|
580
|
+
});
|
|
581
|
+
|
|
582
|
+
it('merges run Langfuse defaults with agent redaction overrides', () => {
|
|
583
|
+
const resolved = resolveLangfuseConfig(
|
|
584
|
+
{
|
|
585
|
+
enabled: true,
|
|
586
|
+
publicKey: 'pk-run',
|
|
587
|
+
secretKey: 'sk-run',
|
|
588
|
+
baseUrl: 'https://langfuse.test',
|
|
589
|
+
toolNodeTracing: { enabled: true },
|
|
590
|
+
toolOutputTracing: {
|
|
591
|
+
enabled: true,
|
|
592
|
+
redactionText: '[redacted]',
|
|
593
|
+
},
|
|
594
|
+
},
|
|
595
|
+
{
|
|
596
|
+
toolOutputTracing: {
|
|
597
|
+
enabled: false,
|
|
598
|
+
redactedToolNames: ['execute_sql'],
|
|
599
|
+
},
|
|
600
|
+
}
|
|
601
|
+
);
|
|
602
|
+
|
|
603
|
+
expect(resolved).toMatchObject({
|
|
604
|
+
enabled: true,
|
|
605
|
+
publicKey: 'pk-run',
|
|
606
|
+
secretKey: 'sk-run',
|
|
607
|
+
baseUrl: 'https://langfuse.test',
|
|
608
|
+
toolNodeTracing: { enabled: true },
|
|
609
|
+
toolOutputTracing: {
|
|
610
|
+
enabled: false,
|
|
611
|
+
redactedToolNames: ['execute_sql'],
|
|
612
|
+
redactionText: '[redacted]',
|
|
613
|
+
},
|
|
614
|
+
});
|
|
615
|
+
});
|
|
616
|
+
});
|
package/src/tools/ToolNode.ts
CHANGED
|
@@ -41,6 +41,7 @@ import {
|
|
|
41
41
|
import { safeDispatchCustomEvent } from '@/utils/events';
|
|
42
42
|
import { executeHooks } from '@/hooks';
|
|
43
43
|
import { toLangChainContent } from '@/messages/langchain';
|
|
44
|
+
import { withLangfuseToolOutputTracingConfig } from '@/langfuseToolOutputTracing';
|
|
44
45
|
import { Constants, GraphEvents, CODE_EXECUTION_TOOLS } from '@/common';
|
|
45
46
|
import {
|
|
46
47
|
buildReferenceKey,
|
|
@@ -102,6 +103,8 @@ type RunToolBatchContext = {
|
|
|
102
103
|
additionalContextsSink?: string[];
|
|
103
104
|
};
|
|
104
105
|
|
|
106
|
+
const TOOL_NODE_RUN_NAME = 'tool_batch';
|
|
107
|
+
|
|
105
108
|
/**
|
|
106
109
|
* Per-batch context for `dispatchToolEvents` / `executeViaEvent`.
|
|
107
110
|
* Mirrors {@link RunToolBatchContext} for the event-driven path,
|
|
@@ -394,6 +397,8 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
|
|
|
394
397
|
private loadRuntimeTools?: t.ToolRefGenerator;
|
|
395
398
|
handleToolErrors = true;
|
|
396
399
|
trace = false;
|
|
400
|
+
private runLangfuse?: t.LangfuseConfig;
|
|
401
|
+
private agentLangfuse?: t.LangfuseConfig;
|
|
397
402
|
toolCallStepIds?: Map<string, string>;
|
|
398
403
|
errorHandler?: t.ToolNodeConstructorParams['errorHandler'];
|
|
399
404
|
private toolUsageCount: Map<string, number>;
|
|
@@ -473,6 +478,9 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
|
|
|
473
478
|
toolMap,
|
|
474
479
|
name,
|
|
475
480
|
tags,
|
|
481
|
+
trace,
|
|
482
|
+
runLangfuse,
|
|
483
|
+
agentLangfuse,
|
|
476
484
|
errorHandler,
|
|
477
485
|
toolCallStepIds,
|
|
478
486
|
handleToolErrors,
|
|
@@ -494,7 +502,14 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
|
|
|
494
502
|
toolExecution,
|
|
495
503
|
fileCheckpointer,
|
|
496
504
|
}: t.ToolNodeConstructorParams) {
|
|
497
|
-
super({
|
|
505
|
+
super({
|
|
506
|
+
name: name ?? TOOL_NODE_RUN_NAME,
|
|
507
|
+
tags,
|
|
508
|
+
func: (input, config) => this.run(input, config),
|
|
509
|
+
});
|
|
510
|
+
this.trace = trace ?? this.trace;
|
|
511
|
+
this.runLangfuse = runLangfuse;
|
|
512
|
+
this.agentLangfuse = agentLangfuse;
|
|
498
513
|
this.toolMap = toolMap ?? new Map(tools.map((tool) => [tool.name, tool]));
|
|
499
514
|
this.toolCallStepIds = toolCallStepIds;
|
|
500
515
|
this.handleToolErrors = handleToolErrors ?? this.handleToolErrors;
|
|
@@ -545,6 +560,19 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
|
|
|
545
560
|
}
|
|
546
561
|
}
|
|
547
562
|
|
|
563
|
+
override async invoke(
|
|
564
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
565
|
+
input: any,
|
|
566
|
+
options?: Partial<RunnableConfig>
|
|
567
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
568
|
+
): Promise<any> {
|
|
569
|
+
return withLangfuseToolOutputTracingConfig(
|
|
570
|
+
this.runLangfuse,
|
|
571
|
+
() => super.invoke(input, options),
|
|
572
|
+
this.agentLangfuse
|
|
573
|
+
);
|
|
574
|
+
}
|
|
575
|
+
|
|
548
576
|
/**
|
|
549
577
|
* Returns the run-scoped tool output registry, or `undefined` when
|
|
550
578
|
* the feature is disabled.
|
|
@@ -2140,13 +2168,12 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
|
|
|
2140
2168
|
|
|
2141
2169
|
/**
|
|
2142
2170
|
* `interrupt()` reads the current `RunnableConfig` from
|
|
2143
|
-
* AsyncLocalStorage
|
|
2144
|
-
*
|
|
2145
|
-
*
|
|
2146
|
-
* `
|
|
2147
|
-
*
|
|
2148
|
-
*
|
|
2149
|
-
* `interrupt()` needs to suspend and resume.
|
|
2171
|
+
* AsyncLocalStorage. ToolNode usually runs with tracing disabled
|
|
2172
|
+
* (unless Langfuse explicitly enables it), so the upstream
|
|
2173
|
+
* `runWithConfig` frame may not exist. Re-anchor here using the
|
|
2174
|
+
* node's own `config` — Pregel hands us a config that already
|
|
2175
|
+
* carries every checkpoint/scratchpad key `interrupt()` needs to
|
|
2176
|
+
* suspend and resume.
|
|
2150
2177
|
*/
|
|
2151
2178
|
const resumeValue = AsyncLocalStorageProviderSingleton.runWithConfig(
|
|
2152
2179
|
config,
|