@traccia2/sdk 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -5
- package/dist/integrations/langchain-callback.d.ts +5 -0
- package/dist/integrations/langchain-callback.d.ts.map +1 -1
- package/dist/integrations/langchain-callback.js +106 -28
- package/dist/integrations/langchain-callback.js.map +1 -1
- package/package.json +6 -1
- package/src/integrations/langchain-callback.ts +112 -31
package/README.md
CHANGED
|
@@ -336,7 +336,7 @@ The SDK provides a `TraciaCallbackHandler` for seamless integration with LangCha
|
|
|
336
336
|
|
|
337
337
|
```typescript
|
|
338
338
|
import { startTracing, getTracer } from '@traccia/sdk';
|
|
339
|
-
import { TraciaCallbackHandler } from '@traccia/sdk/integrations
|
|
339
|
+
import { TraciaCallbackHandler } from '@traccia/sdk/integrations';
|
|
340
340
|
import { ChatOpenAI } from '@langchain/openai';
|
|
341
341
|
import { AgentExecutor, createOpenAIToolsAgent } from 'langchain/agents';
|
|
342
342
|
|
|
@@ -375,28 +375,63 @@ const result = await executor.invoke({
|
|
|
375
375
|
|
|
376
376
|
The `TraciaCallbackHandler` automatically traces:
|
|
377
377
|
|
|
378
|
-
- **LLM Calls**: Model name, token counts, latency
|
|
378
|
+
- **LLM Calls**: Model name, temperature, max tokens, top-p/k, token counts, latency
|
|
379
379
|
- **Agent Actions**: Tool selection and execution
|
|
380
380
|
- **Chain Steps**: Multi-step reasoning and intermediate results
|
|
381
381
|
- **Tool Usage**: Tool names, inputs, and outputs
|
|
382
382
|
- **Errors & Exceptions**: Automatic error recording
|
|
383
383
|
|
|
384
|
+
**Auto-Captured LLM Configuration:**
|
|
385
|
+
|
|
386
|
+
The handler automatically extracts and logs LLM defaults:
|
|
387
|
+
|
|
388
|
+
```typescript
|
|
389
|
+
// For any LLM, these are automatically captured:
|
|
390
|
+
{
|
|
391
|
+
'model': 'gpt-4', // Auto-detected from modelName/model/name
|
|
392
|
+
'temperature': 0.7, // Auto-detected from llm.temperature
|
|
393
|
+
'max_tokens': 2000, // Auto-detected from llm.maxTokens
|
|
394
|
+
'top_p': 0.9, // Auto-detected if available
|
|
395
|
+
'top_k': 40, // Auto-detected if available
|
|
396
|
+
'base_url': 'http://localhost:11434' // For local models (Ollama)
|
|
397
|
+
}
|
|
398
|
+
```
|
|
399
|
+
|
|
384
400
|
**Traced Metrics:**
|
|
385
401
|
|
|
386
402
|
```typescript
|
|
387
403
|
// Automatically captured attributes:
|
|
388
404
|
{
|
|
389
|
-
|
|
405
|
+
// LLM Configuration (auto-extracted from defaults)
|
|
406
|
+
'model': 'gpt-4', // Auto-detected model name
|
|
407
|
+
'temperature': 0.7, // If configured
|
|
408
|
+
'max_tokens': 2000, // If configured
|
|
409
|
+
'top_p': 0.9, // If available
|
|
410
|
+
'top_k': 40, // If available
|
|
411
|
+
'base_url': 'http://localhost:11434' // For local models
|
|
412
|
+
|
|
413
|
+
// LLM Runtime Metrics
|
|
390
414
|
'llm.tokens.prompt': 150,
|
|
391
415
|
'llm.tokens.completion': 75,
|
|
392
416
|
'llm.tokens.total': 225,
|
|
417
|
+
'output_length': 450,
|
|
418
|
+
'finish_reason': 'stop', // If provided by model
|
|
419
|
+
|
|
420
|
+
// Agent & Tool Usage
|
|
393
421
|
'agent.action': 'tool_use',
|
|
394
422
|
'agent.tool': 'calculator',
|
|
395
423
|
'agent.tool.input': '{"expression": "2 + 2"}',
|
|
396
424
|
'agent.tool.output': '4',
|
|
425
|
+
|
|
426
|
+
// Chain Execution
|
|
397
427
|
'chain.steps': 3,
|
|
398
|
-
'
|
|
399
|
-
'
|
|
428
|
+
'chain_type': 'agent_executor',
|
|
429
|
+
'input_length': 1250,
|
|
430
|
+
'output_length': 450,
|
|
431
|
+
|
|
432
|
+
// Error Handling
|
|
433
|
+
'error.type': 'ValidationError', // If applicable
|
|
434
|
+
'error.message': 'Invalid input', // If applicable
|
|
400
435
|
}
|
|
401
436
|
```
|
|
402
437
|
|
|
@@ -24,6 +24,11 @@
|
|
|
24
24
|
export declare class TraciaCallbackHandler {
|
|
25
25
|
private tracer;
|
|
26
26
|
private spanStack;
|
|
27
|
+
/**
|
|
28
|
+
* Extract model name from LLM instance, checking multiple property locations.
|
|
29
|
+
* Different LLM implementations store the model name in different properties.
|
|
30
|
+
*/
|
|
31
|
+
private extractModelName;
|
|
27
32
|
/**
|
|
28
33
|
* Handle LLM start - called when an LLM begins execution.
|
|
29
34
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"langchain-callback.d.ts","sourceRoot":"","sources":["../../src/integrations/langchain-callback.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,SAAS,CAAiC;IAElD;;OAEG;IACU,cAAc,CACzB,GAAG,EAAE,GAAG,EACR,OAAO,EAAE,MAAM,EAAE,EACjB,KAAK,EAAE,MAAM,EACb,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"langchain-callback.d.ts","sourceRoot":"","sources":["../../src/integrations/langchain-callback.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,SAAS,CAAiC;IAElD;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;OAEG;IACU,cAAc,CACzB,GAAG,EAAE,GAAG,EACR,OAAO,EAAE,MAAM,EAAE,EACjB,KAAK,EAAE,MAAM,EACb,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,IAAI,CAAC;IA0ChB;;OAEG;IACU,YAAY,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8CpE;;OAEG;IACU,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASvE;;OAEG;IACU,gBAAgB,CAC3B,KAAK,EAAE,GAAG,EACV,MAAM,EAAE,GAAG,EACX,KAAK,EAAE,MAAM,EACb,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,IAAI,CAAC;IAuBhB;;OAEG;IACU,cAAc,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBtE;;OAEG;IACU,gBAAgB,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASzE;;OAEG;IACU,eAAe,CAC1B,IAAI,EAAE,GAAG,EACT,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,IAAI,CAAC;IAuBhB;;OAEG;IACU,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcxE;;OAEG;IACU,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASxE;;OAEG;IACU,iBAAiB,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzE;;OAEG;IACU,iBAAiB,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAa1E"}
|
|
@@ -30,18 +30,59 @@ class TraciaCallbackHandler {
|
|
|
30
30
|
this.tracer = (0, auto_1.getTracer)('langchain');
|
|
31
31
|
this.spanStack = new Map();
|
|
32
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* Extract model name from LLM instance, checking multiple property locations.
|
|
35
|
+
* Different LLM implementations store the model name in different properties.
|
|
36
|
+
*/
|
|
37
|
+
extractModelName(llm) {
|
|
38
|
+
// Check common model name properties
|
|
39
|
+
if (llm.modelName)
|
|
40
|
+
return llm.modelName; // ChatOpenAI, ChatAnthropic, etc.
|
|
41
|
+
if (llm.model)
|
|
42
|
+
return llm.model; // Ollama, etc.
|
|
43
|
+
if (llm.name && !llm.name.startsWith('langchain'))
|
|
44
|
+
return llm.name; // Generic name
|
|
45
|
+
if (llm._modelType)
|
|
46
|
+
return llm._modelType; // Fallback to type
|
|
47
|
+
if (llm.client?.model)
|
|
48
|
+
return llm.client.model; // Nested model property
|
|
49
|
+
return 'unknown';
|
|
50
|
+
}
|
|
33
51
|
/**
|
|
34
52
|
* Handle LLM start - called when an LLM begins execution.
|
|
35
53
|
*/
|
|
36
54
|
async handleLLMStart(llm, prompts, runId, _parentRunId) {
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
55
|
+
const modelName = this.extractModelName(llm);
|
|
56
|
+
const attributes = {
|
|
57
|
+
type: 'llm',
|
|
58
|
+
model: modelName,
|
|
59
|
+
prompt_count: prompts.length,
|
|
60
|
+
first_prompt_length: prompts[0]?.length || 0,
|
|
61
|
+
};
|
|
62
|
+
// Capture temperature if available
|
|
63
|
+
if (llm.temperature !== undefined) {
|
|
64
|
+
attributes.temperature = llm.temperature;
|
|
65
|
+
}
|
|
66
|
+
// Capture max tokens if available
|
|
67
|
+
if (llm.maxTokens !== undefined) {
|
|
68
|
+
attributes.max_tokens = llm.maxTokens;
|
|
69
|
+
}
|
|
70
|
+
if (llm.max_tokens !== undefined) {
|
|
71
|
+
attributes.max_tokens = llm.max_tokens;
|
|
72
|
+
}
|
|
73
|
+
// Capture top_p if available
|
|
74
|
+
if (llm.topP !== undefined) {
|
|
75
|
+
attributes.top_p = llm.topP;
|
|
76
|
+
}
|
|
77
|
+
// Capture top_k if available
|
|
78
|
+
if (llm.topK !== undefined) {
|
|
79
|
+
attributes.top_k = llm.topK;
|
|
80
|
+
}
|
|
81
|
+
// Capture base URL for local models (Ollama, etc.)
|
|
82
|
+
if (llm.baseUrl) {
|
|
83
|
+
attributes.base_url = llm.baseUrl;
|
|
84
|
+
}
|
|
85
|
+
const span = this.tracer.startSpan('llm', { attributes });
|
|
45
86
|
this.spanStack.set(runId, span);
|
|
46
87
|
}
|
|
47
88
|
/**
|
|
@@ -51,15 +92,36 @@ class TraciaCallbackHandler {
|
|
|
51
92
|
const span = this.spanStack.get(runId);
|
|
52
93
|
if (span) {
|
|
53
94
|
try {
|
|
54
|
-
|
|
95
|
+
// Try multiple ways to get token usage
|
|
96
|
+
const tokenUsage = output?.llmOutput?.token_usage ||
|
|
97
|
+
output?.token_usage ||
|
|
98
|
+
output?.metadata?.token_usage;
|
|
55
99
|
if (tokenUsage) {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
100
|
+
if (tokenUsage.prompt_tokens !== undefined) {
|
|
101
|
+
span.setAttribute('llm.tokens.prompt', tokenUsage.prompt_tokens);
|
|
102
|
+
}
|
|
103
|
+
if (tokenUsage.completion_tokens !== undefined) {
|
|
104
|
+
span.setAttribute('llm.tokens.completion', tokenUsage.completion_tokens);
|
|
105
|
+
}
|
|
106
|
+
const totalTokens = (tokenUsage.prompt_tokens || 0) + (tokenUsage.completion_tokens || 0);
|
|
107
|
+
if (totalTokens > 0) {
|
|
108
|
+
span.setAttribute('llm.tokens.total', totalTokens);
|
|
109
|
+
}
|
|
59
110
|
}
|
|
111
|
+
// Capture output text length
|
|
60
112
|
if (output?.text) {
|
|
61
113
|
span.setAttribute('output_length', output.text.length);
|
|
62
114
|
}
|
|
115
|
+
else if (output?.generations && Array.isArray(output.generations)) {
|
|
116
|
+
const firstGeneration = output.generations[0];
|
|
117
|
+
if (firstGeneration?.[0]?.text) {
|
|
118
|
+
span.setAttribute('output_length', firstGeneration[0].text.length);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
// Capture finish reason if available
|
|
122
|
+
if (output?.llmOutput?.finish_reason) {
|
|
123
|
+
span.setAttribute('finish_reason', output.llmOutput.finish_reason);
|
|
124
|
+
}
|
|
63
125
|
}
|
|
64
126
|
catch (error) {
|
|
65
127
|
// Silently fail on attribute setting
|
|
@@ -84,14 +146,22 @@ class TraciaCallbackHandler {
|
|
|
84
146
|
*/
|
|
85
147
|
async handleChainStart(chain, inputs, runId, _parentRunId) {
|
|
86
148
|
const chainName = chain.name || chain._chainType || 'chain';
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
149
|
+
const attributes = {
|
|
150
|
+
type: 'chain',
|
|
151
|
+
chain_name: chainName,
|
|
152
|
+
chain_type: chain._chainType,
|
|
153
|
+
input_keys: Object.keys(inputs || {}).join(','),
|
|
154
|
+
input_count: Object.keys(inputs || {}).length,
|
|
155
|
+
};
|
|
156
|
+
// Capture total input length
|
|
157
|
+
try {
|
|
158
|
+
const inputStr = JSON.stringify(inputs);
|
|
159
|
+
attributes.input_length = inputStr.length;
|
|
160
|
+
}
|
|
161
|
+
catch (error) {
|
|
162
|
+
// Silently fail
|
|
163
|
+
}
|
|
164
|
+
const span = this.tracer.startSpan(`chain:${chainName}`, { attributes });
|
|
95
165
|
this.spanStack.set(runId, span);
|
|
96
166
|
}
|
|
97
167
|
/**
|
|
@@ -129,14 +199,22 @@ class TraciaCallbackHandler {
|
|
|
129
199
|
*/
|
|
130
200
|
async handleToolStart(tool, input, runId, _parentRunId) {
|
|
131
201
|
const toolName = tool.name || 'unknown-tool';
|
|
132
|
-
const
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
202
|
+
const attributes = {
|
|
203
|
+
type: 'tool',
|
|
204
|
+
tool_name: toolName,
|
|
205
|
+
tool_description: tool.description || '',
|
|
206
|
+
input_length: typeof input === 'string' ? input.length : (typeof input === 'object' ? JSON.stringify(input).length : 0),
|
|
207
|
+
};
|
|
208
|
+
// Try to capture structured input
|
|
209
|
+
try {
|
|
210
|
+
if (typeof input === 'object') {
|
|
211
|
+
attributes.input_keys = Object.keys(input).join(',');
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
catch (error) {
|
|
215
|
+
// Silently fail
|
|
216
|
+
}
|
|
217
|
+
const span = this.tracer.startSpan(`tool:${toolName}`, { attributes });
|
|
140
218
|
this.spanStack.set(runId, span);
|
|
141
219
|
}
|
|
142
220
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"langchain-callback.js","sourceRoot":"","sources":["../../src/integrations/langchain-callback.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;AAGH,kCAAoC;AAEpC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAa,qBAAqB;IAAlC;QACU,WAAM,GAAG,IAAA,gBAAS,EAAC,WAAW,CAAC,CAAC;QAChC,cAAS,GAAuB,IAAI,GAAG,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"langchain-callback.js","sourceRoot":"","sources":["../../src/integrations/langchain-callback.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;AAGH,kCAAoC;AAEpC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAa,qBAAqB;IAAlC;QACU,WAAM,GAAG,IAAA,gBAAS,EAAC,WAAW,CAAC,CAAC;QAChC,cAAS,GAAuB,IAAI,GAAG,EAAE,CAAC;IAwRpD,CAAC;IAtRC;;;OAGG;IACK,gBAAgB,CAAC,GAAQ;QAC/B,qCAAqC;QACrC,IAAI,GAAG,CAAC,SAAS;YAAE,OAAO,GAAG,CAAC,SAAS,CAAC,CAAW,kCAAkC;QACrF,IAAI,GAAG,CAAC,KAAK;YAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAmB,eAAe;QAClE,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,CAAC,CAAE,eAAe;QACpF,IAAI,GAAG,CAAC,UAAU;YAAE,OAAO,GAAG,CAAC,UAAU,CAAC,CAAS,mBAAmB;QACtE,IAAI,GAAG,CAAC,MAAM,EAAE,KAAK;YAAE,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAI,wBAAwB;QAC3E,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,cAAc,CACzB,GAAQ,EACR,OAAiB,EACjB,KAAa,EACb,YAAqB;QAErB,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAE7C,MAAM,UAAU,GAAwB;YACtC,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,SAAS;YAChB,YAAY,EAAE,OAAO,CAAC,MAAM;YAC5B,mBAAmB,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC;SAC7C,CAAC;QAEF,mCAAmC;QACnC,IAAI,GAAG,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YAClC,UAAU,CAAC,WAAW,GAAG,GAAG,CAAC,WAAW,CAAC;QAC3C,CAAC;QAED,kCAAkC;QAClC,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAChC,UAAU,CAAC,UAAU,GAAG,GAAG,CAAC,SAAS,CAAC;QACxC,CAAC;QACD,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACjC,UAAU,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;QACzC,CAAC;QAED,6BAA6B;QAC7B,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC3B,UAAU,CAAC,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC;QAC9B,CAAC;QAED,6BAA6B;QAC7B,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC3B,UAAU,CAAC,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC;QAC9B,CAAC;QAED,mDAAmD;QACnD,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,UAAU,CAAC,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC;QACpC,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,YAAY,CAAC,MAAW,EAAE,KAAa;QAClD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC;gBACH,uCAAuC;gBACvC,MAAM,UAAU,GACd,MAAM,EAAE,SAAS,EAAE,WAAW;oBAC9B,MAAM,EAAE,WAAW;oBACnB,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC;gBAEhC,IAAI,UAAU,EAAE,CAAC;oBACf,IAAI,UAAU,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;wBAC3C,IAAI,CAAC,YAAY,CAAC,mBAAmB,EAAE,UAAU,CAAC,aAAa,CAAC,CAAC;oBACnE,CAAC;oBACD,IAAI,UAAU,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;wBAC/C,IAAI,CAAC,YAAY,CAAC,uBAAuB,EAAE,UAAU,CAAC,iBAAiB,CAAC,CAAC;oBAC3E,CAAC;oBACD,MAAM,WAAW,GAAG,CAAC,UAAU,CAAC,aAAa,IAAI,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,iBAAiB,IAAI,CAAC,CAAC,CAAC;oBAC1F,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;wBACpB,IAAI,CAAC,YAAY,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;oBACrD,CAAC;gBACH,CAAC;gBAED,6BAA6B;gBAC7B,IAAI,MAAM,EAAE,IAAI,EAAE,CAAC;oBACjB,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACzD,CAAC;qBAAM,IAAI,MAAM,EAAE,WAAW,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;oBACpE,MAAM,eAAe,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;oBAC9C,IAAI,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;wBAC/B,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACrE,CAAC;gBACH,CAAC;gBAED,qCAAqC;gBACrC,IAAI,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC;oBACrC,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;gBACrE,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,qCAAqC;YACvC,CAAC;YAED,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,cAAc,CAAC,KAAY,EAAE,KAAa;QACrD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;YACzD,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,gBAAgB,CAC3B,KAAU,EACV,MAAW,EACX,KAAa,EACb,YAAqB;QAErB,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,UAAU,IAAI,OAAO,CAAC;QAE5D,MAAM,UAAU,GAAwB;YACtC,IAAI,EAAE,OAAO;YACb,UAAU,EAAE,SAAS;YACrB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAC/C,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM;SAC9C,CAAC;QAEF,6BAA6B;QAC7B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACxC,UAAU,CAAC,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC5C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gBAAgB;QAClB,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,SAAS,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,cAAc,CAAC,MAAW,EAAE,KAAa;QACpD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC;gBACH,IAAI,MAAM,EAAE,CAAC;oBACX,MAAM,SAAS,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;oBAC/E,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,gBAAgB;YAClB,CAAC;YAED,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,gBAAgB,CAAC,KAAY,EAAE,KAAa;QACvD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC,CAAC;YAC3D,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,eAAe,CAC1B,IAAS,EACT,KAAa,EACb,KAAa,EACb,YAAqB;QAErB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,cAAc,CAAC;QAE7C,MAAM,UAAU,GAAwB;YACtC,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,QAAQ;YACnB,gBAAgB,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE;YACxC,YAAY,EAAE,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;SACxH,CAAC;QAEF,kCAAkC;QAClC,IAAI,CAAC;YACH,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,UAAU,CAAC,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gBAAgB;QAClB,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,aAAa,CAAC,MAAc,EAAE,KAAa;QACtD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC;gBACH,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAC1D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,gBAAgB;YAClB,CAAC;YAED,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,eAAe,CAAC,KAAY,EAAE,KAAa;QACtD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAC1D,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,iBAAiB,CAAC,MAAW,EAAE,KAAa;QACvD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC;gBACH,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,gBAAgB;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,iBAAiB,CAAC,MAAW,EAAE,KAAa;QACvD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC;gBACH,IAAI,CAAC,YAAY,CAAC,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;YAChF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,gBAAgB;YAClB,CAAC;YAED,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;CACF;AA1RD,sDA0RC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@traccia2/sdk",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.4",
|
|
4
4
|
"description": "Tracing SDK for agent applications",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -19,6 +19,11 @@
|
|
|
19
19
|
"import": "./dist/tracer/index.js",
|
|
20
20
|
"require": "./dist/tracer/index.js",
|
|
21
21
|
"types": "./dist/tracer/index.d.ts"
|
|
22
|
+
},
|
|
23
|
+
"./integrations": {
|
|
24
|
+
"import": "./dist/integrations/index.js",
|
|
25
|
+
"require": "./dist/integrations/index.js",
|
|
26
|
+
"types": "./dist/integrations/index.d.ts"
|
|
22
27
|
}
|
|
23
28
|
},
|
|
24
29
|
"files": [
|
|
@@ -29,6 +29,20 @@ export class TraciaCallbackHandler {
|
|
|
29
29
|
private tracer = getTracer('langchain');
|
|
30
30
|
private spanStack: Map<string, ISpan> = new Map();
|
|
31
31
|
|
|
32
|
+
/**
|
|
33
|
+
* Extract model name from LLM instance, checking multiple property locations.
|
|
34
|
+
* Different LLM implementations store the model name in different properties.
|
|
35
|
+
*/
|
|
36
|
+
private extractModelName(llm: any): string {
|
|
37
|
+
// Check common model name properties
|
|
38
|
+
if (llm.modelName) return llm.modelName; // ChatOpenAI, ChatAnthropic, etc.
|
|
39
|
+
if (llm.model) return llm.model; // Ollama, etc.
|
|
40
|
+
if (llm.name && !llm.name.startsWith('langchain')) return llm.name; // Generic name
|
|
41
|
+
if (llm._modelType) return llm._modelType; // Fallback to type
|
|
42
|
+
if (llm.client?.model) return llm.client.model; // Nested model property
|
|
43
|
+
return 'unknown';
|
|
44
|
+
}
|
|
45
|
+
|
|
32
46
|
/**
|
|
33
47
|
* Handle LLM start - called when an LLM begins execution.
|
|
34
48
|
*/
|
|
@@ -38,15 +52,44 @@ export class TraciaCallbackHandler {
|
|
|
38
52
|
runId: string,
|
|
39
53
|
_parentRunId?: string
|
|
40
54
|
): Promise<void> {
|
|
41
|
-
const
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
}
|
|
55
|
+
const modelName = this.extractModelName(llm);
|
|
56
|
+
|
|
57
|
+
const attributes: Record<string, any> = {
|
|
58
|
+
type: 'llm',
|
|
59
|
+
model: modelName,
|
|
60
|
+
prompt_count: prompts.length,
|
|
61
|
+
first_prompt_length: prompts[0]?.length || 0,
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
// Capture temperature if available
|
|
65
|
+
if (llm.temperature !== undefined) {
|
|
66
|
+
attributes.temperature = llm.temperature;
|
|
67
|
+
}
|
|
49
68
|
|
|
69
|
+
// Capture max tokens if available
|
|
70
|
+
if (llm.maxTokens !== undefined) {
|
|
71
|
+
attributes.max_tokens = llm.maxTokens;
|
|
72
|
+
}
|
|
73
|
+
if (llm.max_tokens !== undefined) {
|
|
74
|
+
attributes.max_tokens = llm.max_tokens;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Capture top_p if available
|
|
78
|
+
if (llm.topP !== undefined) {
|
|
79
|
+
attributes.top_p = llm.topP;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Capture top_k if available
|
|
83
|
+
if (llm.topK !== undefined) {
|
|
84
|
+
attributes.top_k = llm.topK;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Capture base URL for local models (Ollama, etc.)
|
|
88
|
+
if (llm.baseUrl) {
|
|
89
|
+
attributes.base_url = llm.baseUrl;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const span = this.tracer.startSpan('llm', { attributes });
|
|
50
93
|
this.spanStack.set(runId, span);
|
|
51
94
|
}
|
|
52
95
|
|
|
@@ -57,18 +100,38 @@ export class TraciaCallbackHandler {
|
|
|
57
100
|
const span = this.spanStack.get(runId);
|
|
58
101
|
if (span) {
|
|
59
102
|
try {
|
|
60
|
-
|
|
103
|
+
// Try multiple ways to get token usage
|
|
104
|
+
const tokenUsage =
|
|
105
|
+
output?.llmOutput?.token_usage ||
|
|
106
|
+
output?.token_usage ||
|
|
107
|
+
output?.metadata?.token_usage;
|
|
108
|
+
|
|
61
109
|
if (tokenUsage) {
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
(
|
|
67
|
-
|
|
110
|
+
if (tokenUsage.prompt_tokens !== undefined) {
|
|
111
|
+
span.setAttribute('llm.tokens.prompt', tokenUsage.prompt_tokens);
|
|
112
|
+
}
|
|
113
|
+
if (tokenUsage.completion_tokens !== undefined) {
|
|
114
|
+
span.setAttribute('llm.tokens.completion', tokenUsage.completion_tokens);
|
|
115
|
+
}
|
|
116
|
+
const totalTokens = (tokenUsage.prompt_tokens || 0) + (tokenUsage.completion_tokens || 0);
|
|
117
|
+
if (totalTokens > 0) {
|
|
118
|
+
span.setAttribute('llm.tokens.total', totalTokens);
|
|
119
|
+
}
|
|
68
120
|
}
|
|
69
121
|
|
|
122
|
+
// Capture output text length
|
|
70
123
|
if (output?.text) {
|
|
71
124
|
span.setAttribute('output_length', output.text.length);
|
|
125
|
+
} else if (output?.generations && Array.isArray(output.generations)) {
|
|
126
|
+
const firstGeneration = output.generations[0];
|
|
127
|
+
if (firstGeneration?.[0]?.text) {
|
|
128
|
+
span.setAttribute('output_length', firstGeneration[0].text.length);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Capture finish reason if available
|
|
133
|
+
if (output?.llmOutput?.finish_reason) {
|
|
134
|
+
span.setAttribute('finish_reason', output.llmOutput.finish_reason);
|
|
72
135
|
}
|
|
73
136
|
} catch (error) {
|
|
74
137
|
// Silently fail on attribute setting
|
|
@@ -101,15 +164,24 @@ export class TraciaCallbackHandler {
|
|
|
101
164
|
_parentRunId?: string
|
|
102
165
|
): Promise<void> {
|
|
103
166
|
const chainName = chain.name || chain._chainType || 'chain';
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
},
|
|
111
|
-
}
|
|
167
|
+
|
|
168
|
+
const attributes: Record<string, any> = {
|
|
169
|
+
type: 'chain',
|
|
170
|
+
chain_name: chainName,
|
|
171
|
+
chain_type: chain._chainType,
|
|
172
|
+
input_keys: Object.keys(inputs || {}).join(','),
|
|
173
|
+
input_count: Object.keys(inputs || {}).length,
|
|
174
|
+
};
|
|
112
175
|
|
|
176
|
+
// Capture total input length
|
|
177
|
+
try {
|
|
178
|
+
const inputStr = JSON.stringify(inputs);
|
|
179
|
+
attributes.input_length = inputStr.length;
|
|
180
|
+
} catch (error) {
|
|
181
|
+
// Silently fail
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const span = this.tracer.startSpan(`chain:${chainName}`, { attributes });
|
|
113
185
|
this.spanStack.set(runId, span);
|
|
114
186
|
}
|
|
115
187
|
|
|
@@ -155,15 +227,24 @@ export class TraciaCallbackHandler {
|
|
|
155
227
|
_parentRunId?: string
|
|
156
228
|
): Promise<void> {
|
|
157
229
|
const toolName = tool.name || 'unknown-tool';
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
230
|
+
|
|
231
|
+
const attributes: Record<string, any> = {
|
|
232
|
+
type: 'tool',
|
|
233
|
+
tool_name: toolName,
|
|
234
|
+
tool_description: tool.description || '',
|
|
235
|
+
input_length: typeof input === 'string' ? input.length : (typeof input === 'object' ? JSON.stringify(input).length : 0),
|
|
236
|
+
};
|
|
237
|
+
|
|
238
|
+
// Try to capture structured input
|
|
239
|
+
try {
|
|
240
|
+
if (typeof input === 'object') {
|
|
241
|
+
attributes.input_keys = Object.keys(input).join(',');
|
|
242
|
+
}
|
|
243
|
+
} catch (error) {
|
|
244
|
+
// Silently fail
|
|
245
|
+
}
|
|
166
246
|
|
|
247
|
+
const span = this.tracer.startSpan(`tool:${toolName}`, { attributes });
|
|
167
248
|
this.spanStack.set(runId, span);
|
|
168
249
|
}
|
|
169
250
|
|