@librechat/agents 3.2.64 → 3.2.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/graphs/Graph.cjs +18 -2
- package/dist/cjs/graphs/Graph.cjs.map +1 -1
- package/dist/cjs/instrumentation.cjs +15 -3
- package/dist/cjs/instrumentation.cjs.map +1 -1
- package/dist/cjs/langfuseToolOutputTracing.cjs +1 -2
- package/dist/cjs/langfuseToolOutputTracing.cjs.map +1 -1
- package/dist/cjs/langfuseTraceShaping.cjs +51 -24
- package/dist/cjs/langfuseTraceShaping.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/index.cjs +73 -9
- package/dist/cjs/llm/anthropic/index.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/types.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/utils/message_inputs.cjs +49 -9
- package/dist/cjs/llm/anthropic/utils/message_inputs.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/utils/message_outputs.cjs +3 -1
- package/dist/cjs/llm/anthropic/utils/message_outputs.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/utils/stream_events.cjs +337 -0
- package/dist/cjs/llm/anthropic/utils/stream_events.cjs.map +1 -0
- package/dist/cjs/llm/bedrock/utils/message_inputs.cjs +8 -0
- package/dist/cjs/llm/bedrock/utils/message_inputs.cjs.map +1 -1
- package/dist/cjs/tools/BashExecutor.cjs +9 -8
- package/dist/cjs/tools/BashExecutor.cjs.map +1 -1
- package/dist/cjs/tools/CodeExecutor.cjs +9 -7
- package/dist/cjs/tools/CodeExecutor.cjs.map +1 -1
- package/dist/cjs/tools/search/crw-scraper.cjs +165 -0
- package/dist/cjs/tools/search/crw-scraper.cjs.map +1 -0
- package/dist/cjs/tools/search/crw-search.cjs +105 -0
- package/dist/cjs/tools/search/crw-search.cjs.map +1 -0
- package/dist/cjs/tools/search/search.cjs +4 -2
- package/dist/cjs/tools/search/search.cjs.map +1 -1
- package/dist/cjs/tools/search/tool.cjs +15 -3
- package/dist/cjs/tools/search/tool.cjs.map +1 -1
- package/dist/esm/graphs/Graph.mjs +18 -2
- package/dist/esm/graphs/Graph.mjs.map +1 -1
- package/dist/esm/instrumentation.mjs +15 -3
- package/dist/esm/instrumentation.mjs.map +1 -1
- package/dist/esm/langfuseToolOutputTracing.mjs +1 -2
- package/dist/esm/langfuseToolOutputTracing.mjs.map +1 -1
- package/dist/esm/langfuseTraceShaping.mjs +51 -24
- package/dist/esm/langfuseTraceShaping.mjs.map +1 -1
- package/dist/esm/llm/anthropic/index.mjs +73 -9
- package/dist/esm/llm/anthropic/index.mjs.map +1 -1
- package/dist/esm/llm/anthropic/types.mjs.map +1 -1
- package/dist/esm/llm/anthropic/utils/message_inputs.mjs +49 -9
- package/dist/esm/llm/anthropic/utils/message_inputs.mjs.map +1 -1
- package/dist/esm/llm/anthropic/utils/message_outputs.mjs +3 -2
- package/dist/esm/llm/anthropic/utils/message_outputs.mjs.map +1 -1
- package/dist/esm/llm/anthropic/utils/stream_events.mjs +337 -0
- package/dist/esm/llm/anthropic/utils/stream_events.mjs.map +1 -0
- package/dist/esm/llm/bedrock/utils/message_inputs.mjs +8 -0
- package/dist/esm/llm/bedrock/utils/message_inputs.mjs.map +1 -1
- package/dist/esm/tools/BashExecutor.mjs +9 -8
- package/dist/esm/tools/BashExecutor.mjs.map +1 -1
- package/dist/esm/tools/CodeExecutor.mjs +9 -7
- package/dist/esm/tools/CodeExecutor.mjs.map +1 -1
- package/dist/esm/tools/search/crw-scraper.mjs +163 -0
- package/dist/esm/tools/search/crw-scraper.mjs.map +1 -0
- package/dist/esm/tools/search/crw-search.mjs +103 -0
- package/dist/esm/tools/search/crw-search.mjs.map +1 -0
- package/dist/esm/tools/search/search.mjs +4 -2
- package/dist/esm/tools/search/search.mjs.map +1 -1
- package/dist/esm/tools/search/tool.mjs +15 -3
- package/dist/esm/tools/search/tool.mjs.map +1 -1
- package/dist/types/graphs/Graph.d.ts +2 -0
- package/dist/types/langfuseTraceShaping.d.ts +12 -6
- package/dist/types/llm/anthropic/index.d.ts +2 -0
- package/dist/types/llm/anthropic/types.d.ts +2 -0
- package/dist/types/llm/anthropic/utils/message_outputs.d.ts +1 -2
- package/dist/types/llm/anthropic/utils/stream_events.d.ts +25 -0
- package/dist/types/tools/BashExecutor.d.ts +7 -6
- package/dist/types/tools/CodeExecutor.d.ts +7 -5
- package/dist/types/tools/search/crw-scraper.d.ts +41 -0
- package/dist/types/tools/search/crw-search.d.ts +4 -0
- package/dist/types/tools/search/types.d.ts +88 -3
- package/dist/types/types/graph.d.ts +10 -3
- package/package.json +1 -1
- package/src/graphs/Graph.ts +36 -3
- package/src/instrumentation.ts +20 -0
- package/src/langfuseToolOutputTracing.ts +2 -4
- package/src/langfuseTraceShaping.ts +73 -20
- package/src/llm/anthropic/index.ts +134 -10
- package/src/llm/anthropic/inherited-content-utils.spec.ts +10 -5
- package/src/llm/anthropic/inherited-stream-events.spec.ts +513 -9
- package/src/llm/anthropic/llm.spec.ts +100 -16
- package/src/llm/anthropic/types.ts +3 -0
- package/src/llm/anthropic/utils/cross-provider-server-tools.test.ts +110 -0
- package/src/llm/anthropic/utils/message_inputs.ts +75 -3
- package/src/llm/anthropic/utils/message_outputs.ts +10 -2
- package/src/llm/anthropic/utils/stream_events.ts +471 -0
- package/src/llm/bedrock/utils/cross-provider-server-tools.test.ts +122 -0
- package/src/llm/bedrock/utils/message_inputs.ts +13 -0
- package/src/specs/langfuse-instrumentation.test.ts +64 -0
- package/src/specs/langfuse-routing.integration.test.ts +49 -4
- package/src/specs/langfuse-tool-output-tracing.test.ts +5 -4
- package/src/specs/langfuse-trace-shaping.test.ts +80 -9
- package/src/specs/subagent.test.ts +180 -0
- package/src/tools/BashExecutor.ts +9 -8
- package/src/tools/CodeExecutor.ts +9 -7
- package/src/tools/__tests__/BashExecutor.test.ts +16 -5
- package/src/tools/__tests__/CodeExecutor.stateful.test.ts +17 -6
- package/src/tools/search/crw-scraper.ts +244 -0
- package/src/tools/search/crw-search.ts +167 -0
- package/src/tools/search/crw.test.ts +836 -0
- package/src/tools/search/search.ts +7 -1
- package/src/tools/search/tool.ts +23 -3
- package/src/tools/search/types.ts +103 -3
- package/src/types/graph.ts +10 -3
|
@@ -25,6 +25,8 @@ type SpanStartRecord = {
|
|
|
25
25
|
name: string;
|
|
26
26
|
params: ProcessorParams;
|
|
27
27
|
traceId: string;
|
|
28
|
+
spanId: string;
|
|
29
|
+
parentSpanId?: string;
|
|
28
30
|
};
|
|
29
31
|
|
|
30
32
|
const spanStarts: SpanStartRecord[] = [];
|
|
@@ -140,11 +142,15 @@ jest.mock('@langfuse/otel', () => ({
|
|
|
140
142
|
LangfuseSpanProcessor: jest.fn().mockImplementation((params) => ({
|
|
141
143
|
forceFlush: jest.fn(),
|
|
142
144
|
onEnd: jest.fn(),
|
|
143
|
-
onStart: jest.fn((span) => {
|
|
145
|
+
onStart: jest.fn((span, parentContext) => {
|
|
146
|
+
const spanContext = span.spanContext();
|
|
147
|
+
const parentSpanId = otelTrace.getSpanContext(parentContext)?.spanId;
|
|
144
148
|
spanStarts.push({
|
|
145
149
|
name: span.name,
|
|
146
150
|
params,
|
|
147
|
-
traceId:
|
|
151
|
+
traceId: spanContext.traceId,
|
|
152
|
+
spanId: spanContext.spanId,
|
|
153
|
+
...(parentSpanId != null ? { parentSpanId } : {}),
|
|
148
154
|
});
|
|
149
155
|
}),
|
|
150
156
|
shutdown: jest.fn(),
|
|
@@ -190,7 +196,6 @@ function tenantLangfuse(tenantId: string): t.LangfuseConfig {
|
|
|
190
196
|
deterministicTraceId: true,
|
|
191
197
|
metadata: { tenantId },
|
|
192
198
|
tags: [`tenant:${tenantId}`],
|
|
193
|
-
toolNodeTracing: { enabled: true },
|
|
194
199
|
toolOutputTracing: { enabled: true },
|
|
195
200
|
};
|
|
196
201
|
}
|
|
@@ -239,6 +244,23 @@ function expectNamedSpansUseTraceId({
|
|
|
239
244
|
}
|
|
240
245
|
}
|
|
241
246
|
|
|
247
|
+
function expectChildSpanParentName({
|
|
248
|
+
starts,
|
|
249
|
+
childName,
|
|
250
|
+
parentNamePrefix,
|
|
251
|
+
}: {
|
|
252
|
+
starts: SpanStartRecord[];
|
|
253
|
+
childName: string;
|
|
254
|
+
parentNamePrefix: string;
|
|
255
|
+
}): void {
|
|
256
|
+
const children = starts.filter((record) => record.name === childName);
|
|
257
|
+
expect(children).not.toHaveLength(0);
|
|
258
|
+
for (const child of children) {
|
|
259
|
+
const parent = starts.find((record) => record.spanId === child.parentSpanId);
|
|
260
|
+
expect(parent?.name.startsWith(parentNamePrefix)).toBe(true);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
242
264
|
function expectOnlyTraceIds(
|
|
243
265
|
starts: SpanStartRecord[],
|
|
244
266
|
allowedTraceIds: string[]
|
|
@@ -446,6 +468,23 @@ describe('Langfuse per-run routing integration', () => {
|
|
|
446
468
|
getChatModelClassSpy.mockRestore();
|
|
447
469
|
});
|
|
448
470
|
|
|
471
|
+
it('keeps tool observations attached to the exported dispatch parent', async () => {
|
|
472
|
+
await runTenantFlow('tenant-hierarchy');
|
|
473
|
+
|
|
474
|
+
const starts = startsForTenant('tenant-hierarchy');
|
|
475
|
+
expect(starts.some((record) => record.name === 'tool_batch')).toBe(false);
|
|
476
|
+
expectChildSpanParentName({
|
|
477
|
+
starts,
|
|
478
|
+
childName: 'echo',
|
|
479
|
+
parentNamePrefix: 'tools=',
|
|
480
|
+
});
|
|
481
|
+
expectChildSpanParentName({
|
|
482
|
+
starts,
|
|
483
|
+
childName: 'subagent',
|
|
484
|
+
parentNamePrefix: 'tools=',
|
|
485
|
+
});
|
|
486
|
+
});
|
|
487
|
+
|
|
449
488
|
it('routes parallel root, model, tool, subagent, and title spans to each run config', async () => {
|
|
450
489
|
await Promise.all([runTenantFlow('tenant-a'), runTenantFlow('tenant-b')]);
|
|
451
490
|
|
|
@@ -462,10 +501,16 @@ describe('Langfuse per-run routing integration', () => {
|
|
|
462
501
|
names: [
|
|
463
502
|
`LibreChat Agent: Parent ${tenantId}`,
|
|
464
503
|
'FakeChatModel',
|
|
465
|
-
'
|
|
504
|
+
'echo',
|
|
466
505
|
'subagent',
|
|
467
506
|
],
|
|
468
507
|
});
|
|
508
|
+
expect(starts.some((record) => record.name === 'tool_batch')).toBe(false);
|
|
509
|
+
expectChildSpanParentName({
|
|
510
|
+
starts,
|
|
511
|
+
childName: 'echo',
|
|
512
|
+
parentNamePrefix: 'tools=',
|
|
513
|
+
});
|
|
469
514
|
expectNamedSpansUseTraceId({
|
|
470
515
|
starts,
|
|
471
516
|
traceId: titleTraceId,
|
|
@@ -118,7 +118,7 @@ describe('Langfuse tool output tracing redaction', () => {
|
|
|
118
118
|
process.env = originalEnv;
|
|
119
119
|
});
|
|
120
120
|
|
|
121
|
-
it('
|
|
121
|
+
it('keeps internal ToolNode batch tracing opt-in', () => {
|
|
122
122
|
delete process.env.LANGFUSE_SECRET_KEY;
|
|
123
123
|
delete process.env.LANGFUSE_PUBLIC_KEY;
|
|
124
124
|
delete process.env.LANGFUSE_BASE_URL;
|
|
@@ -132,7 +132,7 @@ describe('Langfuse tool output tracing redaction', () => {
|
|
|
132
132
|
secretKey: 'sk-run',
|
|
133
133
|
},
|
|
134
134
|
})
|
|
135
|
-
).toBe(
|
|
135
|
+
).toBe(false);
|
|
136
136
|
expect(
|
|
137
137
|
shouldTraceToolNodeForLangfuse({
|
|
138
138
|
agentLangfuse: {
|
|
@@ -149,7 +149,7 @@ describe('Langfuse tool output tracing redaction', () => {
|
|
|
149
149
|
process.env.LANGFUSE_PUBLIC_KEY = 'pk-test';
|
|
150
150
|
process.env.LANGFUSE_BASE_URL = 'https://langfuse.test';
|
|
151
151
|
|
|
152
|
-
expect(shouldTraceToolNodeForLangfuse({})).toBe(
|
|
152
|
+
expect(shouldTraceToolNodeForLangfuse({})).toBe(false);
|
|
153
153
|
expect(
|
|
154
154
|
shouldTraceToolNodeForLangfuse({
|
|
155
155
|
runLangfuse: { toolNodeTracing: { enabled: true } },
|
|
@@ -162,7 +162,7 @@ describe('Langfuse tool output tracing redaction', () => {
|
|
|
162
162
|
).toBe(false);
|
|
163
163
|
});
|
|
164
164
|
|
|
165
|
-
it('lets agent
|
|
165
|
+
it('lets an agent explicitly opt into ToolNode batch tracing', () => {
|
|
166
166
|
delete process.env.LANGFUSE_SECRET_KEY;
|
|
167
167
|
delete process.env.LANGFUSE_PUBLIC_KEY;
|
|
168
168
|
delete process.env.LANGFUSE_BASE_URL;
|
|
@@ -177,6 +177,7 @@ describe('Langfuse tool output tracing redaction', () => {
|
|
|
177
177
|
publicKey: 'pk-agent',
|
|
178
178
|
secretKey: 'sk-agent',
|
|
179
179
|
baseUrl: 'https://langfuse.test',
|
|
180
|
+
toolNodeTracing: { enabled: true },
|
|
180
181
|
},
|
|
181
182
|
})
|
|
182
183
|
).toBe(true);
|
|
@@ -26,6 +26,8 @@ const INPUT = LangfuseOtelSpanAttributes.OBSERVATION_INPUT;
|
|
|
26
26
|
const OUTPUT = LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT;
|
|
27
27
|
const TRACE_INPUT = LangfuseOtelSpanAttributes.TRACE_INPUT;
|
|
28
28
|
const TRACE_OUTPUT = LangfuseOtelSpanAttributes.TRACE_OUTPUT;
|
|
29
|
+
const OBSERVATION_TYPE = LangfuseOtelSpanAttributes.OBSERVATION_TYPE;
|
|
30
|
+
const TRACE_TAGS = LangfuseOtelSpanAttributes.TRACE_TAGS;
|
|
29
31
|
|
|
30
32
|
describe('shouldDropLangfuseSpan', () => {
|
|
31
33
|
it('drops langgraph __start__ seed spans', () => {
|
|
@@ -40,6 +42,7 @@ describe('shouldDropLangfuseSpan', () => {
|
|
|
40
42
|
expect(shouldDropLangfuseSpan('GenerateTitle')).toBe(false);
|
|
41
43
|
expect(shouldDropLangfuseSpan('agent=openAI__gpt-5.4')).toBe(false);
|
|
42
44
|
expect(shouldDropLangfuseSpan('ChatOpenAI')).toBe(false);
|
|
45
|
+
expect(shouldDropLangfuseSpan('tool_batch')).toBe(false);
|
|
43
46
|
});
|
|
44
47
|
});
|
|
45
48
|
|
|
@@ -48,9 +51,10 @@ describe('shapeLangfuseSpan', () => {
|
|
|
48
51
|
const span = createSpan('agent=openAI__gpt-5.4', {}, 'parent-1');
|
|
49
52
|
shapeLangfuseSpan(span);
|
|
50
53
|
expect(span.name).toBe('agent');
|
|
54
|
+
expect(span.attributes[OBSERVATION_TYPE]).toBe('agent');
|
|
51
55
|
});
|
|
52
56
|
|
|
53
|
-
it('
|
|
57
|
+
it('shapes tool nodes as stable dispatch chains with scoped call inputs', () => {
|
|
54
58
|
const messages = [
|
|
55
59
|
{ type: 'human', content: 'hello' },
|
|
56
60
|
{
|
|
@@ -71,13 +75,14 @@ describe('shapeLangfuseSpan', () => {
|
|
|
71
75
|
'parent-1'
|
|
72
76
|
);
|
|
73
77
|
shapeLangfuseSpan(span);
|
|
74
|
-
expect(span.name).toBe('
|
|
78
|
+
expect(span.name).toBe('tool-dispatch');
|
|
79
|
+
expect(span.attributes[OBSERVATION_TYPE]).toBe('chain');
|
|
75
80
|
expect(JSON.parse(span.attributes[INPUT] as string)).toEqual([
|
|
76
81
|
{ name: 'get_service_details', args: { path: 'organizations/1' } },
|
|
77
82
|
]);
|
|
78
83
|
});
|
|
79
84
|
|
|
80
|
-
it('
|
|
85
|
+
it('preserves every pending call in a multi-tool dispatch input', () => {
|
|
81
86
|
const messages = [
|
|
82
87
|
{
|
|
83
88
|
type: 'ai',
|
|
@@ -89,12 +94,16 @@ describe('shapeLangfuseSpan', () => {
|
|
|
89
94
|
},
|
|
90
95
|
];
|
|
91
96
|
const span = createSpan(
|
|
92
|
-
'
|
|
97
|
+
'tools=openAI__gpt-5.4',
|
|
93
98
|
{ [INPUT]: JSON.stringify({ messages }) },
|
|
94
99
|
'parent-1'
|
|
95
100
|
);
|
|
96
101
|
shapeLangfuseSpan(span);
|
|
97
|
-
expect(span.
|
|
102
|
+
expect(JSON.parse(span.attributes[INPUT] as string)).toEqual([
|
|
103
|
+
{ name: 'web_search', args: { q: 'a' } },
|
|
104
|
+
{ name: 'web_search', args: { q: 'b' } },
|
|
105
|
+
{ name: 'execute_code', args: { code: '1+1' } },
|
|
106
|
+
]);
|
|
98
107
|
});
|
|
99
108
|
|
|
100
109
|
it('reads tool calls from serialized langchain message kwargs', () => {
|
|
@@ -115,10 +124,10 @@ describe('shapeLangfuseSpan', () => {
|
|
|
115
124
|
'parent-1'
|
|
116
125
|
);
|
|
117
126
|
shapeLangfuseSpan(span);
|
|
118
|
-
expect(span.name).toBe('
|
|
127
|
+
expect(span.name).toBe('tool-dispatch');
|
|
119
128
|
});
|
|
120
129
|
|
|
121
|
-
it('
|
|
130
|
+
it('keeps a stable tool-dispatch shape when no tool calls are found', () => {
|
|
122
131
|
const original = JSON.stringify({
|
|
123
132
|
messages: [{ type: 'human', content: 'hi' }],
|
|
124
133
|
});
|
|
@@ -128,12 +137,14 @@ describe('shapeLangfuseSpan', () => {
|
|
|
128
137
|
'parent-1'
|
|
129
138
|
);
|
|
130
139
|
shapeLangfuseSpan(span);
|
|
131
|
-
expect(span.name).toBe('
|
|
140
|
+
expect(span.name).toBe('tool-dispatch');
|
|
141
|
+
expect(span.attributes[OBSERVATION_TYPE]).toBe('chain');
|
|
132
142
|
expect(span.attributes[INPUT]).toBe(original);
|
|
133
143
|
});
|
|
134
144
|
|
|
135
145
|
it('sets root span and trace input/output to the question and answer', () => {
|
|
136
146
|
const span = createSpan('LibreChat Agent', {
|
|
147
|
+
[TRACE_TAGS]: JSON.stringify(['librechat', 'agent']),
|
|
137
148
|
[INPUT]: JSON.stringify({
|
|
138
149
|
messages: [
|
|
139
150
|
{ type: 'system', content: 'You are helpful.' },
|
|
@@ -156,6 +167,7 @@ describe('shapeLangfuseSpan', () => {
|
|
|
156
167
|
|
|
157
168
|
it('extracts answer text from content part arrays', () => {
|
|
158
169
|
const span = createSpan('LibreChat Agent', {
|
|
170
|
+
[TRACE_TAGS]: JSON.stringify(['librechat', 'agent']),
|
|
159
171
|
[INPUT]: JSON.stringify([{ type: 'human', content: 'hi' }]),
|
|
160
172
|
[OUTPUT]: JSON.stringify({
|
|
161
173
|
messages: [
|
|
@@ -189,6 +201,65 @@ describe('shapeLangfuseSpan', () => {
|
|
|
189
201
|
const span = createSpan('LibreChat Agent', { [INPUT]: 'plain text' });
|
|
190
202
|
shapeLangfuseSpan(span);
|
|
191
203
|
expect(span.attributes[INPUT]).toBe('plain text');
|
|
192
|
-
expect(span.attributes[TRACE_INPUT]).
|
|
204
|
+
expect(span.attributes[TRACE_INPUT]).toBe('plain text');
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it('renames generation spans to a provider-agnostic name', () => {
|
|
208
|
+
const span = createSpan(
|
|
209
|
+
'ChatOpenAI',
|
|
210
|
+
{ [OBSERVATION_TYPE]: 'generation' },
|
|
211
|
+
'parent-1'
|
|
212
|
+
);
|
|
213
|
+
shapeLangfuseSpan(span);
|
|
214
|
+
expect(span.name).toBe('llm');
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
it('marks only agent-tagged root spans as agent observations', () => {
|
|
218
|
+
const span = createSpan('LibreChat Agent', {
|
|
219
|
+
[TRACE_TAGS]: JSON.stringify(['librechat', 'agent']),
|
|
220
|
+
[INPUT]: JSON.stringify({
|
|
221
|
+
messages: [{ type: 'human', content: 'hi' }],
|
|
222
|
+
}),
|
|
223
|
+
});
|
|
224
|
+
shapeLangfuseSpan(span);
|
|
225
|
+
expect(span.attributes[OBSERVATION_TYPE]).toBe('agent');
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
it('marks title-tagged root spans as chain observations', () => {
|
|
229
|
+
const span = createSpan('LibreChat Title', {
|
|
230
|
+
[TRACE_TAGS]: JSON.stringify(['librechat', 'title']),
|
|
231
|
+
[INPUT]: 'Conversation text',
|
|
232
|
+
[OUTPUT]: 'Conversation title',
|
|
233
|
+
});
|
|
234
|
+
|
|
235
|
+
shapeLangfuseSpan(span);
|
|
236
|
+
|
|
237
|
+
expect(span.attributes[OBSERVATION_TYPE]).toBe('chain');
|
|
238
|
+
expect(span.attributes[TRACE_INPUT]).toBe('Conversation text');
|
|
239
|
+
expect(span.attributes[TRACE_OUTPUT]).toBe('Conversation title');
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
it('does not classify untagged root spans as agents', () => {
|
|
243
|
+
const span = createSpan('Custom root', { [INPUT]: 'input' });
|
|
244
|
+
|
|
245
|
+
shapeLangfuseSpan(span);
|
|
246
|
+
|
|
247
|
+
expect(span.attributes[OBSERVATION_TYPE]).toBeUndefined();
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
it('shapes standalone generation roots without replacing their type', () => {
|
|
251
|
+
const span = createSpan('ChatOpenAI', {
|
|
252
|
+
[OBSERVATION_TYPE]: 'generation',
|
|
253
|
+
[TRACE_TAGS]: JSON.stringify(['librechat', 'title']),
|
|
254
|
+
[INPUT]: 'Generate a title',
|
|
255
|
+
[OUTPUT]: 'A useful title',
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
shapeLangfuseSpan(span);
|
|
259
|
+
|
|
260
|
+
expect(span.name).toBe('llm');
|
|
261
|
+
expect(span.attributes[OBSERVATION_TYPE]).toBe('generation');
|
|
262
|
+
expect(span.attributes[TRACE_INPUT]).toBe('Generate a title');
|
|
263
|
+
expect(span.attributes[TRACE_OUTPUT]).toBe('A useful title');
|
|
193
264
|
});
|
|
194
265
|
});
|
|
@@ -312,6 +312,186 @@ describe('Subagent Integration', () => {
|
|
|
312
312
|
createWorkflowSpy.mockRestore();
|
|
313
313
|
});
|
|
314
314
|
|
|
315
|
+
it('forwards event-driven tools through nested child graphs', async () => {
|
|
316
|
+
const originalCreateWorkflow = StandardGraph.prototype.createWorkflow;
|
|
317
|
+
const parentToolHandler = jest.fn(
|
|
318
|
+
(_event: string, rawData: unknown): void => {
|
|
319
|
+
const request = rawData as t.ToolExecuteBatchRequest;
|
|
320
|
+
request.resolve(
|
|
321
|
+
request.toolCalls.map((call) => ({
|
|
322
|
+
toolCallId: call.id,
|
|
323
|
+
status: 'success' as const,
|
|
324
|
+
content: `ran ${call.name}`,
|
|
325
|
+
}))
|
|
326
|
+
);
|
|
327
|
+
}
|
|
328
|
+
);
|
|
329
|
+
const parentUpdateHandler = jest.fn();
|
|
330
|
+
let specialistToolDefinitions: t.LCTool[] | undefined;
|
|
331
|
+
let forwardedToolResults: t.ToolExecuteResult[] | undefined;
|
|
332
|
+
|
|
333
|
+
const createWorkflowSpy = jest
|
|
334
|
+
.spyOn(StandardGraph.prototype, 'createWorkflow')
|
|
335
|
+
.mockImplementation(function (this: StandardGraph) {
|
|
336
|
+
const workflow = originalCreateWorkflow.call(this);
|
|
337
|
+
if (this.defaultAgentId === 'router') {
|
|
338
|
+
return {
|
|
339
|
+
invoke: jest.fn(async () => {
|
|
340
|
+
const routerContext = this.agentContexts.get('router');
|
|
341
|
+
const nestedTool = (
|
|
342
|
+
routerContext?.graphTools as t.GenericTool[] | undefined
|
|
343
|
+
)?.find(
|
|
344
|
+
(tool) => 'name' in tool && tool.name === Constants.SUBAGENT
|
|
345
|
+
);
|
|
346
|
+
if (nestedTool == null) {
|
|
347
|
+
throw new Error('Nested subagent tool was not created');
|
|
348
|
+
}
|
|
349
|
+
await nestedTool.invoke(
|
|
350
|
+
{
|
|
351
|
+
description: 'Use the event-driven lookup tool.',
|
|
352
|
+
subagent_type: 'specialist',
|
|
353
|
+
},
|
|
354
|
+
callerConfig
|
|
355
|
+
);
|
|
356
|
+
return { messages: [new AIMessage('router done')] };
|
|
357
|
+
}),
|
|
358
|
+
} as unknown as ReturnType<StandardGraph['createWorkflow']>;
|
|
359
|
+
}
|
|
360
|
+
if (this.defaultAgentId === 'specialist') {
|
|
361
|
+
specialistToolDefinitions =
|
|
362
|
+
this.agentContexts.get('specialist')?.toolDefinitions;
|
|
363
|
+
return {
|
|
364
|
+
invoke: jest.fn(async (_state, options) => {
|
|
365
|
+
const invokeOptions = options as
|
|
366
|
+
| { callbacks?: unknown[] }
|
|
367
|
+
| undefined;
|
|
368
|
+
const forwarder = (invokeOptions?.callbacks ?? [])[0] as
|
|
369
|
+
| {
|
|
370
|
+
handleCustomEvent?: (
|
|
371
|
+
eventName: string,
|
|
372
|
+
data: unknown,
|
|
373
|
+
runId: string
|
|
374
|
+
) => Promise<void> | void;
|
|
375
|
+
}
|
|
376
|
+
| undefined;
|
|
377
|
+
if (forwarder?.handleCustomEvent != null) {
|
|
378
|
+
forwardedToolResults = await new Promise<t.ToolExecuteResult[]>(
|
|
379
|
+
(resolve, reject) => {
|
|
380
|
+
const request: t.ToolExecuteBatchRequest = {
|
|
381
|
+
toolCalls: [
|
|
382
|
+
{ id: 'nested-call', name: 'mcp_lookup', args: {} },
|
|
383
|
+
],
|
|
384
|
+
agentId: 'specialist',
|
|
385
|
+
resolve,
|
|
386
|
+
reject,
|
|
387
|
+
};
|
|
388
|
+
void forwarder.handleCustomEvent?.(
|
|
389
|
+
GraphEvents.ON_TOOL_EXECUTE,
|
|
390
|
+
request,
|
|
391
|
+
'specialist-run'
|
|
392
|
+
);
|
|
393
|
+
}
|
|
394
|
+
);
|
|
395
|
+
await forwarder.handleCustomEvent(
|
|
396
|
+
GraphEvents.ON_RUN_STEP,
|
|
397
|
+
{ id: 'specialist-step', type: 'tool_calls' },
|
|
398
|
+
'specialist-run'
|
|
399
|
+
);
|
|
400
|
+
}
|
|
401
|
+
return { messages: [new AIMessage('specialist done')] };
|
|
402
|
+
}),
|
|
403
|
+
} as unknown as ReturnType<StandardGraph['createWorkflow']>;
|
|
404
|
+
}
|
|
405
|
+
return workflow;
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
const rootAgent: t.AgentInputs = {
|
|
409
|
+
agentId: 'root',
|
|
410
|
+
provider: Providers.OPENAI,
|
|
411
|
+
clientOptions: { modelName: 'gpt-4o-mini', apiKey: 'test-key' },
|
|
412
|
+
instructions: 'Delegate through the router.',
|
|
413
|
+
maxContextTokens: 8000,
|
|
414
|
+
maxSubagentDepth: 2,
|
|
415
|
+
subagentConfigs: [
|
|
416
|
+
{
|
|
417
|
+
type: 'router',
|
|
418
|
+
name: 'Router',
|
|
419
|
+
description: 'Routes work to specialists.',
|
|
420
|
+
allowNested: true,
|
|
421
|
+
agentInputs: {
|
|
422
|
+
agentId: 'router',
|
|
423
|
+
provider: Providers.OPENAI,
|
|
424
|
+
clientOptions: { modelName: 'gpt-4o-mini', apiKey: 'test-key' },
|
|
425
|
+
instructions: 'Delegate to the specialist.',
|
|
426
|
+
maxContextTokens: 8000,
|
|
427
|
+
subagentConfigs: [
|
|
428
|
+
{
|
|
429
|
+
type: 'specialist',
|
|
430
|
+
name: 'Specialist',
|
|
431
|
+
description: 'Uses an event-driven tool.',
|
|
432
|
+
agentInputs: {
|
|
433
|
+
agentId: 'specialist',
|
|
434
|
+
provider: Providers.OPENAI,
|
|
435
|
+
clientOptions: {
|
|
436
|
+
modelName: 'gpt-4o-mini',
|
|
437
|
+
apiKey: 'test-key',
|
|
438
|
+
},
|
|
439
|
+
instructions: 'Use the lookup tool.',
|
|
440
|
+
maxContextTokens: 8000,
|
|
441
|
+
toolDefinitions: [{ name: 'mcp_lookup' }],
|
|
442
|
+
},
|
|
443
|
+
},
|
|
444
|
+
],
|
|
445
|
+
},
|
|
446
|
+
},
|
|
447
|
+
],
|
|
448
|
+
};
|
|
449
|
+
|
|
450
|
+
try {
|
|
451
|
+
const run = await Run.create<t.IState>({
|
|
452
|
+
runId: `nested-event-tools-${Date.now()}`,
|
|
453
|
+
graphConfig: { type: 'standard', agents: [rootAgent] },
|
|
454
|
+
customHandlers: {
|
|
455
|
+
[GraphEvents.ON_TOOL_EXECUTE]: { handle: parentToolHandler },
|
|
456
|
+
[GraphEvents.ON_SUBAGENT_UPDATE]: {
|
|
457
|
+
handle: parentUpdateHandler,
|
|
458
|
+
},
|
|
459
|
+
},
|
|
460
|
+
returnContent: true,
|
|
461
|
+
skipCleanup: true,
|
|
462
|
+
});
|
|
463
|
+
const rootContext = (run.Graph as StandardGraph).agentContexts.get(
|
|
464
|
+
'root'
|
|
465
|
+
);
|
|
466
|
+
const rootSubagentTool = (
|
|
467
|
+
rootContext?.graphTools as t.GenericTool[] | undefined
|
|
468
|
+
)?.find((tool) => 'name' in tool && tool.name === Constants.SUBAGENT);
|
|
469
|
+
expect(rootSubagentTool).toBeDefined();
|
|
470
|
+
|
|
471
|
+
await rootSubagentTool!.invoke(
|
|
472
|
+
{ description: 'Route this task.', subagent_type: 'router' },
|
|
473
|
+
callerConfig
|
|
474
|
+
);
|
|
475
|
+
|
|
476
|
+
expect(specialistToolDefinitions).toEqual([{ name: 'mcp_lookup' }]);
|
|
477
|
+
expect(parentToolHandler).toHaveBeenCalledTimes(1);
|
|
478
|
+
expect(forwardedToolResults).toEqual([
|
|
479
|
+
{
|
|
480
|
+
toolCallId: 'nested-call',
|
|
481
|
+
status: 'success',
|
|
482
|
+
content: 'ran mcp_lookup',
|
|
483
|
+
},
|
|
484
|
+
]);
|
|
485
|
+
const forwardedSubagentTypes = parentUpdateHandler.mock.calls.map(
|
|
486
|
+
([, data]) => (data as t.SubagentUpdateEvent).subagentType
|
|
487
|
+
);
|
|
488
|
+
expect(forwardedSubagentTypes).toContain('router');
|
|
489
|
+
expect(forwardedSubagentTypes).not.toContain('specialist');
|
|
490
|
+
} finally {
|
|
491
|
+
createWorkflowSpy.mockRestore();
|
|
492
|
+
}
|
|
493
|
+
});
|
|
494
|
+
|
|
315
495
|
it('should not create subagent tool when maxSubagentDepth is 0', async () => {
|
|
316
496
|
const agentWithZeroDepth: t.AgentInputs = {
|
|
317
497
|
...createParentAgent(),
|
|
@@ -58,17 +58,18 @@ Usage:
|
|
|
58
58
|
`.trim();
|
|
59
59
|
|
|
60
60
|
/**
|
|
61
|
-
* Bash statefulness is filesystem-tier
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
61
|
+
* Bash statefulness is filesystem-tier and scoped to `/mnt/data`. The machine
|
|
62
|
+
* is warm across calls, but each call runs in a fresh sandbox (new process
|
|
63
|
+
* tree + private /tmp), so background processes are reaped when the call ends
|
|
64
|
+
* and anything written outside /mnt/data is discarded. The note must not
|
|
65
|
+
* promise otherwise: a model told background processes survive will start a
|
|
66
|
+
* server in one call and assume it is listening in the next.
|
|
66
67
|
*/
|
|
67
68
|
export const STATEFUL_BASH_NOTE =
|
|
68
|
-
'Session state
|
|
69
|
+
'Session state: commands in this conversation run on the same warm machine, so files written to /mnt/data persist between calls. Each call runs in a fresh, isolated sandbox: shell variables, the working directory, /tmp, and background processes do NOT survive after the call returns — a process started in one call is terminated when that call ends. Only /mnt/data is durable (the machine itself may also be reset at any time).';
|
|
69
70
|
|
|
70
71
|
export const StatefulBashExecutionToolDescription = `
|
|
71
|
-
Runs bash commands and returns stdout/stderr output
|
|
72
|
+
Runs bash commands and returns stdout/stderr output. Commands in this conversation share one warm machine with a persistent /mnt/data, but each command runs in its own isolated sandbox (not a persistent shell session).
|
|
72
73
|
|
|
73
74
|
${STATEFUL_BASH_NOTE}
|
|
74
75
|
|
|
@@ -125,7 +126,7 @@ export function buildBashExecutionToolDescription(options?: {
|
|
|
125
126
|
const STATELESS_BASH_PARAM_NOTE =
|
|
126
127
|
'The environment is stateless; variables and state don\'t persist between executions.';
|
|
127
128
|
const STATEFUL_BASH_PARAM_NOTE =
|
|
128
|
-
'Files
|
|
129
|
+
'Files written to /mnt/data persist between calls on the same warm machine. Each call runs in a fresh sandbox: shell variables, cwd, /tmp, and background processes do NOT survive the call. Only /mnt/data is durable.';
|
|
129
130
|
|
|
130
131
|
export function buildBashExecutionToolSchema(opts?: {
|
|
131
132
|
statefulSessions?: boolean;
|
|
@@ -150,16 +150,18 @@ Usage:
|
|
|
150
150
|
`.trim();
|
|
151
151
|
|
|
152
152
|
/**
|
|
153
|
-
*
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
153
|
+
* Statefulness here is FILESYSTEM-tier, not runtime-tier. Executions in a
|
|
154
|
+
* session reuse one warm machine, so `/mnt/data` carries across calls — but
|
|
155
|
+
* every execution is a brand-new interpreter process in a fresh sandbox, so
|
|
156
|
+
* variables and imports never survive. The note must not imply otherwise: a
|
|
157
|
+
* model told its in-memory state persists writes `df = ...` in one call and
|
|
158
|
+
* `df.head()` in the next, then hits a NameError it was told to treat as rare.
|
|
157
159
|
*/
|
|
158
160
|
export const STATEFUL_ENV_NOTE =
|
|
159
|
-
'Session state
|
|
161
|
+
'Session state: executions in this conversation run on the same warm machine, so files persist between calls — but each execution is a NEW process. Variables, imports, and in-memory data NEVER carry over: every call must re-import and rebuild the state it needs. Only /mnt/data is durable (the machine itself may also be reset at any time), so write anything that must survive there and read it back next call.';
|
|
160
162
|
|
|
161
163
|
export const StatefulCodeExecutionToolDescription = `
|
|
162
|
-
Runs code and returns stdout/stderr output
|
|
164
|
+
Runs code and returns stdout/stderr output. Executions in this conversation share one warm machine with a persistent /mnt/data, but each execution runs as a separate process (not a notebook-style kernel).
|
|
163
165
|
|
|
164
166
|
${STATEFUL_ENV_NOTE}
|
|
165
167
|
|
|
@@ -181,7 +183,7 @@ export function buildCodeExecutionToolDescription(opts?: {
|
|
|
181
183
|
const STATELESS_CODE_PARAM_NOTE =
|
|
182
184
|
'The environment is stateless; variables and imports don\'t persist between executions.';
|
|
183
185
|
const STATEFUL_CODE_PARAM_NOTE =
|
|
184
|
-
'Executions in this conversation
|
|
186
|
+
'Executions in this conversation share one warm machine, so files written to /mnt/data persist between calls. Each execution is a new process: variables and imports do NOT carry over — re-import and reload from /mnt/data every call.';
|
|
185
187
|
|
|
186
188
|
export function buildCodeExecutionToolSchema(opts?: {
|
|
187
189
|
statefulSessions?: boolean;
|
|
@@ -65,13 +65,24 @@ describe('buildBashExecutionToolDescription', () => {
|
|
|
65
65
|
).toBe(StatefulBashExecutionToolDescription);
|
|
66
66
|
});
|
|
67
67
|
|
|
68
|
-
|
|
68
|
+
/* Filesystem-tier only: each call runs in a fresh sandbox (new process
|
|
69
|
+
* tree + private /tmp), so background processes are reaped and non-
|
|
70
|
+
* /mnt/data writes are discarded. The description must not promise
|
|
71
|
+
* otherwise. */
|
|
72
|
+
it('promises /mnt/data persistence WITHOUT promising surviving processes or /tmp', () => {
|
|
69
73
|
const d = StatefulBashExecutionToolDescription;
|
|
70
|
-
expect(d).toContain('
|
|
71
|
-
expect(d).toContain('may be reset');
|
|
74
|
+
expect(d).toContain('same warm machine');
|
|
72
75
|
expect(d).toContain('Only /mnt/data is durable');
|
|
73
|
-
|
|
74
|
-
expect(d).toContain('
|
|
76
|
+
expect(d).toContain('background processes do NOT survive');
|
|
77
|
+
expect(d).toContain('/tmp');
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('never claims /tmp or background processes persist between calls', () => {
|
|
81
|
+
const d = StatefulBashExecutionToolDescription;
|
|
82
|
+
expect(d).not.toContain('files (including /tmp)');
|
|
83
|
+
expect(d).not.toContain(
|
|
84
|
+
'background processes from earlier calls typically persist'
|
|
85
|
+
);
|
|
75
86
|
});
|
|
76
87
|
|
|
77
88
|
it('keeps the artifact-path guidance in both variants', () => {
|
|
@@ -41,22 +41,33 @@ describe('CodeExecutor stateful description', () => {
|
|
|
41
41
|
);
|
|
42
42
|
});
|
|
43
43
|
|
|
44
|
-
|
|
44
|
+
/* Statefulness is filesystem-tier only: the machine is warm across calls,
|
|
45
|
+
* but every execution is a new interpreter process, so in-memory state never
|
|
46
|
+
* carries over. The description must not imply a notebook-style kernel. */
|
|
47
|
+
it('promises filesystem persistence WITHOUT promising a shared runtime', () => {
|
|
45
48
|
const d = StatefulCodeExecutionToolDescription;
|
|
46
|
-
expect(d).toContain('
|
|
47
|
-
expect(d).toContain('
|
|
48
|
-
expect(d).toContain('
|
|
49
|
+
expect(d).toContain('same warm machine');
|
|
50
|
+
expect(d).toContain('/mnt/data');
|
|
51
|
+
expect(d).toContain('NEW process');
|
|
52
|
+
expect(d).toContain('NEVER carry over');
|
|
49
53
|
expect(d).toContain(CODE_ARTIFACT_PATH_GUIDANCE);
|
|
50
54
|
});
|
|
51
55
|
|
|
56
|
+
it('never claims variables/imports survive between executions', () => {
|
|
57
|
+
const d = StatefulCodeExecutionToolDescription;
|
|
58
|
+
expect(d).not.toContain('share one runtime');
|
|
59
|
+
expect(d).not.toContain('typically still available');
|
|
60
|
+
});
|
|
61
|
+
|
|
52
62
|
it('adjusts the code-param note per mode', () => {
|
|
53
63
|
const stateless =
|
|
54
64
|
buildCodeExecutionToolSchema().properties.code.description;
|
|
55
65
|
const stateful = buildCodeExecutionToolSchema({ statefulSessions: true })
|
|
56
66
|
.properties.code.description;
|
|
57
67
|
expect(stateless).toContain('variables and imports don\'t persist');
|
|
58
|
-
expect(stateful).toContain('
|
|
59
|
-
expect(stateful).toContain('
|
|
68
|
+
expect(stateful).toContain('do NOT carry over');
|
|
69
|
+
expect(stateful).toContain('/mnt/data');
|
|
70
|
+
expect(stateful).not.toContain('typically still defined');
|
|
60
71
|
});
|
|
61
72
|
});
|
|
62
73
|
|