@traccia2/sdk 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/exporter/http-exporter.d.ts +8 -2
- package/dist/exporter/http-exporter.d.ts.map +1 -1
- package/dist/exporter/http-exporter.js +51 -12
- package/dist/exporter/http-exporter.js.map +1 -1
- package/dist/integrations/index.d.ts +1 -0
- package/dist/integrations/index.d.ts.map +1 -1
- package/dist/integrations/index.js +3 -1
- package/dist/integrations/index.js.map +1 -1
- package/dist/integrations/langchain-callback.d.ts +57 -91
- package/dist/integrations/langchain-callback.d.ts.map +1 -1
- package/dist/integrations/langchain-callback.js +464 -316
- package/dist/integrations/langchain-callback.js.map +1 -1
- package/dist/integrations/langchain-callback.old.d.ts +96 -0
- package/dist/integrations/langchain-callback.old.d.ts.map +1 -0
- package/dist/integrations/langchain-callback.old.js +371 -0
- package/dist/integrations/langchain-callback.old.js.map +1 -0
- package/dist/tracer/provider.d.ts +3 -3
- package/dist/tracer/provider.d.ts.map +1 -1
- package/dist/tracer/provider.js +9 -0
- package/dist/tracer/provider.js.map +1 -1
- package/dist/types.d.ts +4 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/__tests__/integrations-langchain.test.ts +354 -340
- package/src/exporter/http-exporter.ts +57 -13
- package/src/integrations/index.ts +1 -0
- package/src/integrations/langchain-callback.old.ts +438 -0
- package/src/integrations/langchain-callback.ts +723 -351
- package/src/tracer/provider.ts +9 -4
- package/src/types.ts +4 -0
|
@@ -1,370 +1,518 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* LangChain callback handler for automatic tracing.
|
|
4
|
-
* Integrates with LangChain's callback system to automatically instrument
|
|
5
|
-
* LLM calls, chains, agents, and tools.
|
|
6
|
-
*/
|
|
7
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
3
|
exports.TracciaCallbackHandler = void 0;
|
|
9
|
-
const auto_1 = require("../auto");
|
|
10
4
|
const base_1 = require("@langchain/core/callbacks/base");
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
* Automatically traces LLM calls, chains, agents, and tools.
|
|
14
|
-
*
|
|
15
|
-
* Extends LangChain's BaseCallbackHandler for proper interface compliance.
|
|
16
|
-
* Compatible with LangChain 0.0.x, 0.1.x, 0.2.x, and 1.x versions.
|
|
17
|
-
*
|
|
18
|
-
* @example
|
|
19
|
-
* import { ChatOpenAI } from '@langchain/openai';
|
|
20
|
-
* import { TracciaCallbackHandler } from '@traccia/sdk/integrations';
|
|
21
|
-
*
|
|
22
|
-
* const handler = new TracciaCallbackHandler();
|
|
23
|
-
* const model = new ChatOpenAI({ callbacks: [handler] });
|
|
24
|
-
*
|
|
25
|
-
* const response = await model.invoke({ input: 'Hello!' });
|
|
26
|
-
* // Automatically traced with spans for LLM calls, tokens, latency, etc.
|
|
27
|
-
*/
|
|
5
|
+
const messages_1 = require("@langchain/core/messages");
|
|
6
|
+
const auto_1 = require("../auto");
|
|
28
7
|
class TracciaCallbackHandler extends base_1.BaseCallbackHandler {
|
|
29
|
-
constructor() {
|
|
30
|
-
super(
|
|
31
|
-
this.name =
|
|
8
|
+
constructor(params) {
|
|
9
|
+
super();
|
|
10
|
+
this.name = "TracciaCallbackHandler";
|
|
32
11
|
this.tracer = (0, auto_1.getTracer)('langchain');
|
|
33
|
-
this.
|
|
34
|
-
this.
|
|
12
|
+
this.tags = [];
|
|
13
|
+
this.completionStartTimes = {};
|
|
14
|
+
this.runMap = new Map();
|
|
15
|
+
this.last_trace_id = null;
|
|
16
|
+
this.tags = params?.tags ?? [];
|
|
17
|
+
this.promptToParentRunMap = new Map();
|
|
35
18
|
}
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
if (llm.modelName)
|
|
43
|
-
return llm.modelName; // ChatOpenAI, ChatAnthropic, etc.
|
|
44
|
-
if (llm.model)
|
|
45
|
-
return llm.model; // Ollama, etc.
|
|
46
|
-
if (llm.name && !llm.name.startsWith('langchain'))
|
|
47
|
-
return llm.name; // Generic name
|
|
48
|
-
if (llm._modelType)
|
|
49
|
-
return llm._modelType; // Fallback to type
|
|
50
|
-
if (llm.client?.model)
|
|
51
|
-
return llm.client.model; // Nested model property
|
|
52
|
-
return 'unknown';
|
|
53
|
-
}
|
|
54
|
-
/**
|
|
55
|
-
* Handle LLM start - called when an LLM begins execution.
|
|
56
|
-
*/
|
|
57
|
-
async handleLLMStart(llm, prompts, runId, _parentRunId) {
|
|
58
|
-
const modelName = this.extractModelName(llm);
|
|
59
|
-
const attributes = {
|
|
60
|
-
type: 'llm',
|
|
61
|
-
model: modelName,
|
|
62
|
-
prompt_count: prompts.length,
|
|
63
|
-
first_prompt_length: prompts[0]?.length || 0,
|
|
64
|
-
};
|
|
65
|
-
// Capture temperature if available
|
|
66
|
-
if (llm.temperature !== undefined) {
|
|
67
|
-
attributes.temperature = llm.temperature;
|
|
68
|
-
}
|
|
69
|
-
// Capture max tokens if available
|
|
70
|
-
if (llm.maxTokens !== undefined) {
|
|
71
|
-
attributes.max_tokens = llm.maxTokens;
|
|
72
|
-
}
|
|
73
|
-
if (llm.max_tokens !== undefined) {
|
|
74
|
-
attributes.max_tokens = llm.max_tokens;
|
|
75
|
-
}
|
|
76
|
-
// Capture top_p if available
|
|
77
|
-
if (llm.topP !== undefined) {
|
|
78
|
-
attributes.top_p = llm.topP;
|
|
79
|
-
}
|
|
80
|
-
// Capture top_k if available
|
|
81
|
-
if (llm.topK !== undefined) {
|
|
82
|
-
attributes.top_k = llm.topK;
|
|
83
|
-
}
|
|
84
|
-
// Capture base URL for local models (Ollama, etc.)
|
|
85
|
-
if (llm.baseUrl) {
|
|
86
|
-
attributes.base_url = llm.baseUrl;
|
|
87
|
-
}
|
|
88
|
-
const span = this.tracer.startSpan('llm', { attributes });
|
|
89
|
-
this.spanStack.set(runId, span);
|
|
19
|
+
async handleLLMNewToken(_token, _idx, runId, _parentRunId, _tags, _fields) {
|
|
20
|
+
// if this is the first token, add it to completionStartTimes
|
|
21
|
+
if (runId && !(runId in this.completionStartTimes)) {
|
|
22
|
+
console.debug(`LLM first streaming token: ${runId}`);
|
|
23
|
+
this.completionStartTimes[runId] = new Date();
|
|
24
|
+
}
|
|
90
25
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
if (totalTokens !== undefined) {
|
|
116
|
-
span.setAttribute('llm.tokens.total', totalTokens);
|
|
117
|
-
}
|
|
118
|
-
// Handle detailed token breakdown for models like GPT-4o vision
|
|
119
|
-
// input_token_details contains breakdown of prompt token usage
|
|
120
|
-
if (tokenUsage.input_token_details && typeof tokenUsage.input_token_details === 'object') {
|
|
121
|
-
for (const [key, value] of Object.entries(tokenUsage.input_token_details)) {
|
|
122
|
-
if (typeof value === 'number') {
|
|
123
|
-
span.setAttribute(`llm.tokens.input_${key}`, value);
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
// output_token_details contains breakdown of completion token usage
|
|
128
|
-
if (tokenUsage.output_token_details && typeof tokenUsage.output_token_details === 'object') {
|
|
129
|
-
for (const [key, value] of Object.entries(tokenUsage.output_token_details)) {
|
|
130
|
-
if (typeof value === 'number') {
|
|
131
|
-
span.setAttribute(`llm.tokens.output_${key}`, value);
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
// Capture output text length
|
|
137
|
-
if (output?.text) {
|
|
138
|
-
span.setAttribute('output_length', output.text.length);
|
|
139
|
-
}
|
|
140
|
-
else if (output?.generations && Array.isArray(output.generations)) {
|
|
141
|
-
const firstGeneration = output.generations[0];
|
|
142
|
-
if (firstGeneration?.[0]?.text) {
|
|
143
|
-
span.setAttribute('output_length', firstGeneration[0].text.length);
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
else if (output?.message?.content) {
|
|
147
|
-
// Ollama format (uses message.content)
|
|
148
|
-
const content = output.message.content;
|
|
149
|
-
const contentStr = typeof content === 'string' ? content : JSON.stringify(content);
|
|
150
|
-
span.setAttribute('output_length', contentStr.length);
|
|
151
|
-
}
|
|
152
|
-
else if (typeof output === 'string') {
|
|
153
|
-
// Direct string output
|
|
154
|
-
span.setAttribute('output_length', output.length);
|
|
155
|
-
}
|
|
156
|
-
// Capture finish reason if available
|
|
157
|
-
if (output?.llmOutput?.finish_reason) {
|
|
158
|
-
span.setAttribute('finish_reason', output.llmOutput.finish_reason);
|
|
159
|
-
}
|
|
26
|
+
async handleChainStart(chain, inputs, runId, parentRunId, tags, metadata, _runType, name) {
|
|
27
|
+
try {
|
|
28
|
+
console.debug(`Chain start with Id: ${runId}`);
|
|
29
|
+
this.tags;
|
|
30
|
+
const runName = name ?? chain.id.at(-1)?.toString() ?? "Langchain Run";
|
|
31
|
+
this.registerLangfusePrompt(parentRunId, metadata);
|
|
32
|
+
// In chains, inputs can be a string or an array of BaseMessage
|
|
33
|
+
let finalInput = inputs;
|
|
34
|
+
if (typeof inputs === "object" &&
|
|
35
|
+
"input" in inputs &&
|
|
36
|
+
Array.isArray(inputs["input"]) &&
|
|
37
|
+
inputs["input"].every((m) => m instanceof messages_1.BaseMessage)) {
|
|
38
|
+
finalInput = inputs["input"].map((m) => this.extractChatMessageContent(m));
|
|
39
|
+
}
|
|
40
|
+
else if (typeof inputs === "object" &&
|
|
41
|
+
"messages" in inputs &&
|
|
42
|
+
Array.isArray(inputs["messages"]) &&
|
|
43
|
+
inputs["messages"].every((m) => m instanceof messages_1.BaseMessage)) {
|
|
44
|
+
finalInput = inputs["messages"].map((m) => this.extractChatMessageContent(m));
|
|
45
|
+
}
|
|
46
|
+
else if (typeof inputs === "object" &&
|
|
47
|
+
"content" in inputs &&
|
|
48
|
+
typeof inputs["content"] === "string") {
|
|
49
|
+
finalInput = inputs["content"];
|
|
160
50
|
}
|
|
161
|
-
|
|
162
|
-
|
|
51
|
+
this.startAndRegisterSpan({
|
|
52
|
+
runName,
|
|
53
|
+
parentRunId,
|
|
54
|
+
runId,
|
|
55
|
+
tags,
|
|
56
|
+
metadata,
|
|
57
|
+
attributes: {
|
|
58
|
+
input: finalInput,
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
if (!parentRunId) {
|
|
62
|
+
// Here we would update the trace, but traccia sdk doesn't support it yet
|
|
163
63
|
}
|
|
164
|
-
|
|
165
|
-
|
|
64
|
+
}
|
|
65
|
+
catch (e) {
|
|
66
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
166
67
|
}
|
|
167
68
|
}
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
69
|
+
async handleAgentAction(action, runId, parentRunId) {
|
|
70
|
+
try {
|
|
71
|
+
console.debug(`Agent action ${action.tool} with ID: ${runId}`);
|
|
72
|
+
this.startAndRegisterSpan({
|
|
73
|
+
runId,
|
|
74
|
+
parentRunId,
|
|
75
|
+
runName: action.tool,
|
|
76
|
+
attributes: {
|
|
77
|
+
input: action,
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
catch (e) {
|
|
82
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
177
83
|
}
|
|
178
84
|
}
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
85
|
+
async handleAgentEnd(action, runId, _parentRunId) {
|
|
86
|
+
try {
|
|
87
|
+
console.debug(`Agent finish with ID: ${runId}`);
|
|
88
|
+
this.handleSpanEnd({
|
|
89
|
+
runId,
|
|
90
|
+
attributes: { output: action },
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
catch (e) {
|
|
94
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
async handleChainError(err, runId, _parentRunId) {
|
|
98
|
+
try {
|
|
99
|
+
console.debug(`Chain error: ${err} with ID: ${runId}`);
|
|
100
|
+
const azureRefusalError = this.parseAzureRefusalError(err);
|
|
101
|
+
this.handleSpanEnd({
|
|
102
|
+
runId,
|
|
103
|
+
attributes: {
|
|
104
|
+
level: "ERROR",
|
|
105
|
+
statusMessage: err.toString() + azureRefusalError,
|
|
106
|
+
},
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
catch (e) {
|
|
110
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
async handleGenerationStart(llm, messages, runId, parentRunId, extraParams, tags, metadata, name) {
|
|
114
|
+
console.debug(`Generation start with ID: ${runId} and parentRunId ${parentRunId}`);
|
|
115
|
+
const runName = name ?? llm.id.at(-1)?.toString() ?? "Langchain Generation";
|
|
116
|
+
const modelParameters = {};
|
|
117
|
+
const invocationParams = extraParams?.["invocation_params"];
|
|
118
|
+
for (const [key, value] of Object.entries({
|
|
119
|
+
temperature: invocationParams?.temperature,
|
|
120
|
+
max_tokens: invocationParams?.max_tokens,
|
|
121
|
+
top_p: invocationParams?.top_p,
|
|
122
|
+
frequency_penalty: invocationParams?.frequency_penalty,
|
|
123
|
+
presence_penalty: invocationParams?.presence_penalty,
|
|
124
|
+
request_timeout: invocationParams?.request_timeout,
|
|
125
|
+
})) {
|
|
126
|
+
if (value !== undefined && value !== null) {
|
|
127
|
+
modelParameters[key] = value;
|
|
195
128
|
}
|
|
196
129
|
}
|
|
130
|
+
let extractedModelName;
|
|
131
|
+
if (extraParams) {
|
|
132
|
+
const invocationParamsModelName = extraParams.invocation_params.model;
|
|
133
|
+
const metadataModelName = metadata && "ls_model_name" in metadata
|
|
134
|
+
? metadata["ls_model_name"]
|
|
135
|
+
: undefined;
|
|
136
|
+
extractedModelName = invocationParamsModelName ?? metadataModelName;
|
|
137
|
+
}
|
|
138
|
+
const registeredPrompt = this.promptToParentRunMap.get(parentRunId ?? "root");
|
|
139
|
+
if (registeredPrompt && parentRunId) {
|
|
140
|
+
this.deregisterLangfusePrompt(parentRunId);
|
|
141
|
+
}
|
|
142
|
+
this.startAndRegisterSpan({
|
|
143
|
+
runId,
|
|
144
|
+
parentRunId,
|
|
145
|
+
metadata,
|
|
146
|
+
tags,
|
|
147
|
+
runName,
|
|
148
|
+
attributes: {
|
|
149
|
+
input: messages,
|
|
150
|
+
model: extractedModelName,
|
|
151
|
+
modelParameters: modelParameters,
|
|
152
|
+
prompt: registeredPrompt,
|
|
153
|
+
},
|
|
154
|
+
});
|
|
197
155
|
}
|
|
198
|
-
|
|
199
|
-
* Handle chain start - called when a chain begins execution.
|
|
200
|
-
*/
|
|
201
|
-
async handleChainStart(chain, inputs, runId, _parentRunId) {
|
|
202
|
-
const chainName = chain.name || chain._chainType || 'chain';
|
|
203
|
-
const attributes = {
|
|
204
|
-
type: 'chain',
|
|
205
|
-
chain_name: chainName,
|
|
206
|
-
chain_type: chain._chainType,
|
|
207
|
-
input_keys: Object.keys(inputs || {}).join(','),
|
|
208
|
-
input_count: Object.keys(inputs || {}).length,
|
|
209
|
-
};
|
|
210
|
-
// Capture total input length
|
|
156
|
+
async handleChatModelStart(llm, messages, runId, parentRunId, extraParams, tags, metadata, name) {
|
|
211
157
|
try {
|
|
212
|
-
|
|
213
|
-
|
|
158
|
+
console.debug(`Chat model start with ID: ${runId}`);
|
|
159
|
+
const prompts = messages.flatMap((message) => message.map((m) => this.extractChatMessageContent(m)));
|
|
160
|
+
this.handleGenerationStart(llm, prompts, runId, parentRunId, extraParams, tags, metadata, name);
|
|
214
161
|
}
|
|
215
|
-
catch (
|
|
216
|
-
|
|
162
|
+
catch (e) {
|
|
163
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
217
164
|
}
|
|
218
|
-
const span = this.tracer.startSpan(`chain:${chainName}`, { attributes });
|
|
219
|
-
this.spanStack.set(runId, span);
|
|
220
165
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
const outputStr = typeof output === 'string' ? output : JSON.stringify(output);
|
|
230
|
-
span.setAttribute('output_length', outputStr.length);
|
|
231
|
-
}
|
|
166
|
+
async handleChainEnd(outputs, runId, _parentRunId) {
|
|
167
|
+
try {
|
|
168
|
+
console.debug(`Chain end with ID: ${runId}`);
|
|
169
|
+
let finalOutput = outputs;
|
|
170
|
+
if (typeof outputs === "object" &&
|
|
171
|
+
"output" in outputs &&
|
|
172
|
+
typeof outputs["output"] === "string") {
|
|
173
|
+
finalOutput = outputs["output"];
|
|
232
174
|
}
|
|
233
|
-
|
|
234
|
-
|
|
175
|
+
else if (typeof outputs === "object" &&
|
|
176
|
+
"messages" in outputs &&
|
|
177
|
+
Array.isArray(outputs["messages"]) &&
|
|
178
|
+
outputs["messages"].every((m) => m instanceof messages_1.BaseMessage)) {
|
|
179
|
+
finalOutput = {
|
|
180
|
+
messages: outputs.messages.map((message) => this.extractChatMessageContent(message)),
|
|
181
|
+
};
|
|
235
182
|
}
|
|
236
|
-
|
|
237
|
-
|
|
183
|
+
this.handleSpanEnd({
|
|
184
|
+
runId,
|
|
185
|
+
attributes: {
|
|
186
|
+
output: finalOutput,
|
|
187
|
+
},
|
|
188
|
+
});
|
|
189
|
+
this.deregisterLangfusePrompt(runId);
|
|
190
|
+
}
|
|
191
|
+
catch (e) {
|
|
192
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
238
193
|
}
|
|
239
194
|
}
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
span.end();
|
|
248
|
-
this.spanStack.delete(runId);
|
|
195
|
+
async handleLLMStart(llm, prompts, runId, parentRunId, extraParams, tags, metadata, name) {
|
|
196
|
+
try {
|
|
197
|
+
console.debug(`LLM start with ID: ${runId}`);
|
|
198
|
+
this.handleGenerationStart(llm, prompts, runId, parentRunId, extraParams, tags, metadata, name);
|
|
199
|
+
}
|
|
200
|
+
catch (e) {
|
|
201
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
249
202
|
}
|
|
250
203
|
}
|
|
251
|
-
|
|
252
|
-
* Handle tool start - called when a tool is invoked.
|
|
253
|
-
*/
|
|
254
|
-
async handleToolStart(tool, input, runId, _parentRunId) {
|
|
255
|
-
const toolName = tool.name || 'unknown-tool';
|
|
256
|
-
const attributes = {
|
|
257
|
-
type: 'tool',
|
|
258
|
-
tool_name: toolName,
|
|
259
|
-
tool_description: tool.description || '',
|
|
260
|
-
input_length: typeof input === 'string' ? input.length : (typeof input === 'object' ? JSON.stringify(input).length : 0),
|
|
261
|
-
};
|
|
262
|
-
// Try to capture structured input
|
|
204
|
+
async handleToolStart(tool, input, runId, parentRunId, tags, metadata, name) {
|
|
263
205
|
try {
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
206
|
+
console.debug(`Tool start with ID: ${runId}`);
|
|
207
|
+
this.startAndRegisterSpan({
|
|
208
|
+
runId,
|
|
209
|
+
parentRunId,
|
|
210
|
+
runName: name ?? tool.id.at(-1)?.toString() ?? "Tool execution",
|
|
211
|
+
attributes: {
|
|
212
|
+
input,
|
|
213
|
+
},
|
|
214
|
+
metadata,
|
|
215
|
+
tags,
|
|
216
|
+
});
|
|
267
217
|
}
|
|
268
|
-
catch (
|
|
269
|
-
|
|
218
|
+
catch (e) {
|
|
219
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
270
220
|
}
|
|
271
|
-
const span = this.tracer.startSpan(`tool:${toolName}`, { attributes });
|
|
272
|
-
this.spanStack.set(runId, span);
|
|
273
221
|
}
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
222
|
+
async handleRetrieverStart(retriever, query, runId, parentRunId, tags, metadata, name) {
|
|
223
|
+
try {
|
|
224
|
+
console.debug(`Retriever start with ID: ${runId}`);
|
|
225
|
+
this.startAndRegisterSpan({
|
|
226
|
+
runId,
|
|
227
|
+
parentRunId,
|
|
228
|
+
runName: name ?? retriever.id.at(-1)?.toString() ?? "Retriever",
|
|
229
|
+
attributes: {
|
|
230
|
+
input: query,
|
|
231
|
+
},
|
|
232
|
+
tags,
|
|
233
|
+
metadata,
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
catch (e) {
|
|
237
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
288
238
|
}
|
|
289
239
|
}
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
240
|
+
async handleRetrieverEnd(documents, runId, _parentRunId) {
|
|
241
|
+
try {
|
|
242
|
+
console.debug(`Retriever end with ID: ${runId}`);
|
|
243
|
+
this.handleSpanEnd({
|
|
244
|
+
runId,
|
|
245
|
+
attributes: {
|
|
246
|
+
output: documents,
|
|
247
|
+
},
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
catch (e) {
|
|
251
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
299
252
|
}
|
|
300
253
|
}
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
254
|
+
async handleRetrieverError(err, runId, _parentRunId) {
|
|
255
|
+
try {
|
|
256
|
+
console.debug(`Retriever error: ${err} with ID: ${runId}`);
|
|
257
|
+
this.handleSpanEnd({
|
|
258
|
+
runId,
|
|
259
|
+
attributes: {
|
|
260
|
+
level: "ERROR",
|
|
261
|
+
statusMessage: err.toString(),
|
|
262
|
+
},
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
catch (e) {
|
|
266
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
313
267
|
}
|
|
314
268
|
}
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
269
|
+
async handleToolEnd(output, runId, _parentRunId) {
|
|
270
|
+
try {
|
|
271
|
+
console.debug(`Tool end with ID: ${runId}`);
|
|
272
|
+
this.handleSpanEnd({
|
|
273
|
+
runId,
|
|
274
|
+
attributes: { output },
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
catch (e) {
|
|
278
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
async handleToolError(err, runId, _parentRunId) {
|
|
282
|
+
try {
|
|
283
|
+
console.debug(`Tool error ${err} with ID: ${runId}`);
|
|
284
|
+
this.handleSpanEnd({
|
|
285
|
+
runId,
|
|
286
|
+
attributes: {
|
|
287
|
+
level: "ERROR",
|
|
288
|
+
statusMessage: err.toString(),
|
|
289
|
+
},
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
catch (e) {
|
|
293
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
async handleLLMEnd(output, runId, _parentRunId) {
|
|
297
|
+
try {
|
|
298
|
+
console.debug(`LLM end with ID: ${runId}`);
|
|
299
|
+
const lastResponse = output.generations[output.generations.length - 1][output.generations[output.generations.length - 1].length - 1];
|
|
300
|
+
const llmUsage = this.extractUsageMetadata(lastResponse) ??
|
|
301
|
+
output.llmOutput?.["tokenUsage"];
|
|
302
|
+
const modelName = this.extractModelNameFromMetadata(lastResponse);
|
|
303
|
+
const usageDetails = {
|
|
304
|
+
input: llmUsage?.input_tokens ??
|
|
305
|
+
("promptTokens" in llmUsage ? llmUsage?.promptTokens : undefined),
|
|
306
|
+
output: llmUsage?.output_tokens ??
|
|
307
|
+
("completionTokens" in llmUsage
|
|
308
|
+
? llmUsage?.completionTokens
|
|
309
|
+
: undefined),
|
|
310
|
+
total: llmUsage?.total_tokens ??
|
|
311
|
+
("totalTokens" in llmUsage ? llmUsage?.totalTokens : undefined),
|
|
312
|
+
};
|
|
313
|
+
if (llmUsage && "input_token_details" in llmUsage) {
|
|
314
|
+
for (const [key, val] of Object.entries(llmUsage["input_token_details"] ?? {})) {
|
|
315
|
+
usageDetails[`input_${key}`] = val;
|
|
316
|
+
if ("input" in usageDetails && typeof val === "number") {
|
|
317
|
+
usageDetails["input"] = Math.max(0, usageDetails["input"] - val);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
323
320
|
}
|
|
324
|
-
|
|
325
|
-
|
|
321
|
+
if (llmUsage && "output_token_details" in llmUsage) {
|
|
322
|
+
for (const [key, val] of Object.entries(llmUsage["output_token_details"] ?? {})) {
|
|
323
|
+
usageDetails[`output_${key}`] = val;
|
|
324
|
+
if ("output" in usageDetails && typeof val === "number") {
|
|
325
|
+
usageDetails["output"] = Math.max(0, usageDetails["output"] - val);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
const extractedOutput = "message" in lastResponse
|
|
330
|
+
? this.extractChatMessageContent(lastResponse["message"])
|
|
331
|
+
: lastResponse.text;
|
|
332
|
+
this.handleSpanEnd({
|
|
333
|
+
runId,
|
|
334
|
+
attributes: {
|
|
335
|
+
model: modelName,
|
|
336
|
+
output: extractedOutput,
|
|
337
|
+
completionStartTime: runId in this.completionStartTimes
|
|
338
|
+
? this.completionStartTimes[runId]
|
|
339
|
+
: undefined,
|
|
340
|
+
usageDetails: usageDetails,
|
|
341
|
+
},
|
|
342
|
+
});
|
|
343
|
+
if (runId in this.completionStartTimes) {
|
|
344
|
+
delete this.completionStartTimes[runId];
|
|
326
345
|
}
|
|
327
|
-
|
|
328
|
-
|
|
346
|
+
}
|
|
347
|
+
catch (e) {
|
|
348
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
329
349
|
}
|
|
330
350
|
}
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
351
|
+
async handleLLMError(err, runId, _parentRunId) {
|
|
352
|
+
try {
|
|
353
|
+
console.debug(`LLM error ${err} with ID: ${runId}`);
|
|
354
|
+
const azureRefusalError = this.parseAzureRefusalError(err);
|
|
355
|
+
this.handleSpanEnd({
|
|
356
|
+
runId,
|
|
357
|
+
attributes: {
|
|
358
|
+
level: "ERROR",
|
|
359
|
+
statusMessage: err.toString() + azureRefusalError,
|
|
360
|
+
},
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
catch (e) {
|
|
364
|
+
console.debug(e instanceof Error ? e.message : String(e));
|
|
365
|
+
}
|
|
338
366
|
}
|
|
339
|
-
|
|
340
|
-
|
|
367
|
+
registerLangfusePrompt(parentRunId, metadata) {
|
|
368
|
+
if (metadata && "langfusePrompt" in metadata && parentRunId) {
|
|
369
|
+
this.promptToParentRunMap.set(parentRunId, metadata.langfusePrompt);
|
|
370
|
+
}
|
|
341
371
|
}
|
|
342
|
-
|
|
343
|
-
|
|
372
|
+
deregisterLangfusePrompt(runId) {
|
|
373
|
+
this.promptToParentRunMap.delete(runId);
|
|
344
374
|
}
|
|
345
|
-
|
|
346
|
-
|
|
375
|
+
startAndRegisterSpan(params) {
|
|
376
|
+
const { runName, runId, attributes, metadata, tags } = params;
|
|
377
|
+
const span = this.tracer.startSpan(runName, {
|
|
378
|
+
attributes: {
|
|
379
|
+
...attributes,
|
|
380
|
+
...this.joinTagsAndMetaData(tags, metadata)
|
|
381
|
+
}
|
|
382
|
+
});
|
|
383
|
+
this.runMap.set(runId, span);
|
|
384
|
+
return span;
|
|
347
385
|
}
|
|
348
|
-
|
|
349
|
-
|
|
386
|
+
handleSpanEnd(params) {
|
|
387
|
+
const { runId, attributes = {} } = params;
|
|
388
|
+
const span = this.runMap.get(runId);
|
|
389
|
+
if (!span) {
|
|
390
|
+
console.warn("Span not found in runMap. Skipping operation");
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
for (const [key, value] of Object.entries(attributes)) {
|
|
394
|
+
span.setAttribute(key, value);
|
|
395
|
+
}
|
|
396
|
+
span.end();
|
|
397
|
+
this.runMap.delete(runId);
|
|
350
398
|
}
|
|
351
|
-
|
|
352
|
-
|
|
399
|
+
parseAzureRefusalError(err) {
|
|
400
|
+
let azureRefusalError = "";
|
|
401
|
+
if (typeof err == "object" && "error" in err) {
|
|
402
|
+
try {
|
|
403
|
+
azureRefusalError =
|
|
404
|
+
"\n\nError details:\n" + JSON.stringify(err["error"], null, 2);
|
|
405
|
+
}
|
|
406
|
+
catch { }
|
|
407
|
+
}
|
|
408
|
+
return azureRefusalError;
|
|
353
409
|
}
|
|
354
|
-
|
|
355
|
-
|
|
410
|
+
joinTagsAndMetaData(tags, metadata1, metadata2) {
|
|
411
|
+
const finalDict = {};
|
|
412
|
+
if (tags && tags.length > 0) {
|
|
413
|
+
finalDict.tags = tags;
|
|
414
|
+
}
|
|
415
|
+
if (metadata1) {
|
|
416
|
+
Object.assign(finalDict, metadata1);
|
|
417
|
+
}
|
|
418
|
+
if (metadata2) {
|
|
419
|
+
Object.assign(finalDict, metadata2);
|
|
420
|
+
}
|
|
421
|
+
return this.stripLangfuseKeysFromMetadata(finalDict);
|
|
356
422
|
}
|
|
357
|
-
|
|
358
|
-
|
|
423
|
+
stripLangfuseKeysFromMetadata(metadata) {
|
|
424
|
+
if (!metadata) {
|
|
425
|
+
return;
|
|
426
|
+
}
|
|
427
|
+
const langfuseKeys = [
|
|
428
|
+
"langfusePrompt",
|
|
429
|
+
"langfuseUserId",
|
|
430
|
+
"langfuseSessionId",
|
|
431
|
+
];
|
|
432
|
+
return Object.fromEntries(Object.entries(metadata).filter(([key, _]) => !langfuseKeys.includes(key)));
|
|
359
433
|
}
|
|
360
|
-
|
|
361
|
-
|
|
434
|
+
extractUsageMetadata(generation) {
|
|
435
|
+
try {
|
|
436
|
+
const usageMetadata = "message" in generation &&
|
|
437
|
+
(generation["message"] instanceof messages_1.AIMessage ||
|
|
438
|
+
generation["message"] instanceof messages_1.AIMessageChunk)
|
|
439
|
+
? generation["message"].usage_metadata
|
|
440
|
+
: undefined;
|
|
441
|
+
return usageMetadata;
|
|
442
|
+
}
|
|
443
|
+
catch (err) {
|
|
444
|
+
console.debug(`Error extracting usage metadata: ${err}`);
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
362
447
|
}
|
|
363
|
-
|
|
364
|
-
|
|
448
|
+
extractModelNameFromMetadata(generation) {
|
|
449
|
+
try {
|
|
450
|
+
return "message" in generation &&
|
|
451
|
+
(generation["message"] instanceof messages_1.AIMessage ||
|
|
452
|
+
generation["message"] instanceof messages_1.AIMessageChunk)
|
|
453
|
+
? generation["message"].response_metadata.model_name
|
|
454
|
+
: undefined;
|
|
455
|
+
}
|
|
456
|
+
catch {
|
|
457
|
+
return undefined;
|
|
458
|
+
}
|
|
365
459
|
}
|
|
366
|
-
|
|
367
|
-
|
|
460
|
+
extractChatMessageContent(message) {
|
|
461
|
+
let response = undefined;
|
|
462
|
+
if (message.getType() === "human") {
|
|
463
|
+
response = { content: message.content, role: "user" };
|
|
464
|
+
}
|
|
465
|
+
else if (message.getType() === "generic") {
|
|
466
|
+
response = {
|
|
467
|
+
content: message.content,
|
|
468
|
+
role: "human",
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
else if (message.getType() === "ai") {
|
|
472
|
+
response = { content: message.content, role: "assistant" };
|
|
473
|
+
if ("tool_calls" in message &&
|
|
474
|
+
Array.isArray(message.tool_calls) &&
|
|
475
|
+
(message.tool_calls?.length ?? 0) > 0) {
|
|
476
|
+
response[`tool_calls`] = message[`tool_calls`];
|
|
477
|
+
}
|
|
478
|
+
if ("additional_kwargs" in message &&
|
|
479
|
+
"tool_calls" in message["additional_kwargs"]) {
|
|
480
|
+
response[`tool_calls`] =
|
|
481
|
+
message["additional_kwargs"]["tool_calls"];
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
else if (message.getType() === "system") {
|
|
485
|
+
response = { content: message.content, role: "system" };
|
|
486
|
+
}
|
|
487
|
+
else if (message.getType() === "function") {
|
|
488
|
+
response = {
|
|
489
|
+
content: message.content,
|
|
490
|
+
additional_kwargs: message.additional_kwargs,
|
|
491
|
+
role: message.name,
|
|
492
|
+
};
|
|
493
|
+
}
|
|
494
|
+
else if (message.getType() === "tool") {
|
|
495
|
+
response = {
|
|
496
|
+
content: message.content,
|
|
497
|
+
additional_kwargs: message.additional_kwargs,
|
|
498
|
+
role: message.name,
|
|
499
|
+
};
|
|
500
|
+
}
|
|
501
|
+
else if (!message.name) {
|
|
502
|
+
response = { content: message.content };
|
|
503
|
+
}
|
|
504
|
+
else {
|
|
505
|
+
response = {
|
|
506
|
+
role: message.name,
|
|
507
|
+
content: message.content,
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
if ((message.additional_kwargs.function_call ||
|
|
511
|
+
message.additional_kwargs.tool_calls) &&
|
|
512
|
+
response[`tool_calls`] === undefined) {
|
|
513
|
+
return { ...response, additional_kwargs: message.additional_kwargs };
|
|
514
|
+
}
|
|
515
|
+
return response;
|
|
368
516
|
}
|
|
369
517
|
}
|
|
370
518
|
exports.TracciaCallbackHandler = TracciaCallbackHandler;
|