@node-llm/monitor-otel 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -0
- package/dist/index.d.ts +116 -0
- package/dist/index.js +325 -0
- package/package.json +26 -0
- package/src/NodeLLMSpanProcessor.ts +257 -0
- package/src/index.ts +1 -0
- package/src/types.ts +141 -0
- package/src/utils.ts +314 -0
- package/test/NodeLLMSpanProcessor.test.ts +165 -0
- package/test/fixtures/vercel-ai-spans.ts +379 -0
- package/test/integration.test.ts +432 -0
- package/test/utils.test.ts +278 -0
- package/tsconfig.json +7 -0
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Integration tests for NodeLLMSpanProcessor
|
|
3
|
+
*
|
|
4
|
+
* These tests use recorded span fixtures from Vercel AI SDK
|
|
5
|
+
* to verify the processor handles real-world span data correctly.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { describe, it, expect, beforeEach, vi } from "vitest";
|
|
9
|
+
import { NodeLLMSpanProcessor } from "../src/NodeLLMSpanProcessor.js";
|
|
10
|
+
import { MemoryAdapter, type MonitoringEvent } from "@node-llm/monitor";
|
|
11
|
+
import {
|
|
12
|
+
openaiGenerateText,
|
|
13
|
+
openaiStreamText,
|
|
14
|
+
openaiGenerateObject,
|
|
15
|
+
anthropicGenerateText,
|
|
16
|
+
anthropicGenAiFormat,
|
|
17
|
+
geminiGenerateText,
|
|
18
|
+
toolCallWeather,
|
|
19
|
+
toolCallDatabase,
|
|
20
|
+
errorRateLimit,
|
|
21
|
+
errorContextLength,
|
|
22
|
+
internalDoGenerate,
|
|
23
|
+
httpRequest,
|
|
24
|
+
openaiEmbed,
|
|
25
|
+
openaiEmbedMany,
|
|
26
|
+
allValidSpans,
|
|
27
|
+
filteredOutSpans
|
|
28
|
+
} from "./fixtures/vercel-ai-spans.js";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Helper to get trace summaries from listTraces result
|
|
32
|
+
*/
|
|
33
|
+
async function getTraceSummaries(store: MemoryAdapter, options = {}) {
|
|
34
|
+
const result = await store.listTraces(options);
|
|
35
|
+
return result.items;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Helper to get full event by requestId
|
|
40
|
+
*/
|
|
41
|
+
async function getFullEvent(
|
|
42
|
+
store: MemoryAdapter,
|
|
43
|
+
requestId: string
|
|
44
|
+
): Promise<MonitoringEvent | undefined> {
|
|
45
|
+
const events = await store.getEvents(requestId);
|
|
46
|
+
return events[0];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Helper to get the first full event from the store
|
|
51
|
+
*/
|
|
52
|
+
async function getFirstEvent(store: MemoryAdapter): Promise<MonitoringEvent | undefined> {
|
|
53
|
+
const traces = await getTraceSummaries(store);
|
|
54
|
+
if (traces.length === 0) return undefined;
|
|
55
|
+
return getFullEvent(store, traces[0].requestId);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
describe("NodeLLMSpanProcessor Integration", () => {
|
|
59
|
+
let store: MemoryAdapter;
|
|
60
|
+
let processor: NodeLLMSpanProcessor;
|
|
61
|
+
|
|
62
|
+
beforeEach(() => {
|
|
63
|
+
store = new MemoryAdapter();
|
|
64
|
+
processor = new NodeLLMSpanProcessor(store);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
describe("Provider Normalization", () => {
|
|
68
|
+
it("should normalize openai.responses to openai", async () => {
|
|
69
|
+
processor.onEnd(openaiGenerateText);
|
|
70
|
+
await processor.forceFlush();
|
|
71
|
+
|
|
72
|
+
const traces = await getTraceSummaries(store);
|
|
73
|
+
expect(traces).toHaveLength(1);
|
|
74
|
+
expect(traces[0].provider).toBe("openai");
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it("should normalize openai.chat to openai", async () => {
|
|
78
|
+
processor.onEnd(openaiStreamText);
|
|
79
|
+
await processor.forceFlush();
|
|
80
|
+
|
|
81
|
+
const traces = await getTraceSummaries(store);
|
|
82
|
+
expect(traces[0].provider).toBe("openai");
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it("should normalize anthropic.messages to anthropic", async () => {
|
|
86
|
+
processor.onEnd(anthropicGenerateText);
|
|
87
|
+
await processor.forceFlush();
|
|
88
|
+
|
|
89
|
+
const traces = await getTraceSummaries(store);
|
|
90
|
+
expect(traces[0].provider).toBe("anthropic");
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it("should normalize google.generativeai to google", async () => {
|
|
94
|
+
processor.onEnd(geminiGenerateText);
|
|
95
|
+
await processor.forceFlush();
|
|
96
|
+
|
|
97
|
+
const traces = await getTraceSummaries(store);
|
|
98
|
+
expect(traces[0].provider).toBe("google");
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
describe("Model Extraction", () => {
|
|
103
|
+
it("should extract correct model from OpenAI spans", async () => {
|
|
104
|
+
processor.onEnd(openaiGenerateText);
|
|
105
|
+
await processor.forceFlush();
|
|
106
|
+
|
|
107
|
+
const traces = await getTraceSummaries(store);
|
|
108
|
+
expect(traces[0].model).toBe("gpt-4o-mini");
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it("should extract correct model from Anthropic spans", async () => {
|
|
112
|
+
processor.onEnd(anthropicGenerateText);
|
|
113
|
+
await processor.forceFlush();
|
|
114
|
+
|
|
115
|
+
const traces = await getTraceSummaries(store);
|
|
116
|
+
expect(traces[0].model).toBe("claude-3-5-sonnet-20241022");
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it("should extract correct model from genAi format spans", async () => {
|
|
120
|
+
processor.onEnd(anthropicGenAiFormat);
|
|
121
|
+
await processor.forceFlush();
|
|
122
|
+
|
|
123
|
+
const traces = await getTraceSummaries(store);
|
|
124
|
+
expect(traces[0].model).toBe("claude-3-opus");
|
|
125
|
+
});
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
describe("Token Usage", () => {
|
|
129
|
+
it("should extract prompt and completion tokens from Vercel AI SDK format", async () => {
|
|
130
|
+
processor.onEnd(openaiGenerateText);
|
|
131
|
+
await processor.forceFlush();
|
|
132
|
+
|
|
133
|
+
const event = await getFirstEvent(store);
|
|
134
|
+
expect(event).toBeDefined();
|
|
135
|
+
expect(event!.payload.usage).toEqual({
|
|
136
|
+
promptTokens: 12,
|
|
137
|
+
completionTokens: 28,
|
|
138
|
+
totalTokens: 40
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it("should extract tokens from genAi format", async () => {
|
|
143
|
+
processor.onEnd(anthropicGenAiFormat);
|
|
144
|
+
await processor.forceFlush();
|
|
145
|
+
|
|
146
|
+
const event = await getFirstEvent(store);
|
|
147
|
+
expect(event).toBeDefined();
|
|
148
|
+
expect(event!.payload.usage).toEqual({
|
|
149
|
+
promptTokens: 234,
|
|
150
|
+
completionTokens: 567,
|
|
151
|
+
totalTokens: 801
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
describe("Streaming Metrics", () => {
|
|
157
|
+
it("should capture streaming metrics from streamText spans", async () => {
|
|
158
|
+
processor.onEnd(openaiStreamText);
|
|
159
|
+
await processor.forceFlush();
|
|
160
|
+
|
|
161
|
+
const event = await getFirstEvent(store);
|
|
162
|
+
expect(event).toBeDefined();
|
|
163
|
+
expect(event!.payload.msToFirstChunk).toBe(145);
|
|
164
|
+
expect(event!.payload.msToFinish).toBe(2340);
|
|
165
|
+
expect(event!.payload.avgCompletionTokensPerSecond).toBe(180.5);
|
|
166
|
+
});
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
describe("Custom Metadata", () => {
|
|
170
|
+
it("should extract telemetry metadata", async () => {
|
|
171
|
+
processor.onEnd(openaiStreamText);
|
|
172
|
+
await processor.forceFlush();
|
|
173
|
+
|
|
174
|
+
const event = await getFirstEvent(store);
|
|
175
|
+
expect(event).toBeDefined();
|
|
176
|
+
expect(event!.payload.metadata).toEqual({
|
|
177
|
+
userId: "user-123",
|
|
178
|
+
environment: "production"
|
|
179
|
+
});
|
|
180
|
+
});
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
describe("Content Capture", () => {
|
|
184
|
+
it("should capture prompt and response when enabled", async () => {
|
|
185
|
+
processor.onEnd(openaiGenerateText);
|
|
186
|
+
await processor.forceFlush();
|
|
187
|
+
|
|
188
|
+
const event = await getFirstEvent(store);
|
|
189
|
+
expect(event).toBeDefined();
|
|
190
|
+
expect(event!.payload.prompt).toBe("Hello, how are you?");
|
|
191
|
+
expect(event!.payload.result).toBe(
|
|
192
|
+
"I'm doing well, thank you for asking! How can I help you today?"
|
|
193
|
+
);
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
it("should capture structured object responses", async () => {
|
|
197
|
+
processor.onEnd(openaiGenerateObject);
|
|
198
|
+
await processor.forceFlush();
|
|
199
|
+
|
|
200
|
+
const event = await getFirstEvent(store);
|
|
201
|
+
expect(event).toBeDefined();
|
|
202
|
+
expect(event!.payload.object).toEqual({ name: "John Smith", age: 32 });
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
it("should NOT capture content when disabled", async () => {
|
|
206
|
+
processor = new NodeLLMSpanProcessor(store, { captureContent: false });
|
|
207
|
+
processor.onEnd(openaiGenerateText);
|
|
208
|
+
await processor.forceFlush();
|
|
209
|
+
|
|
210
|
+
const event = await getFirstEvent(store);
|
|
211
|
+
expect(event).toBeDefined();
|
|
212
|
+
expect(event!.payload.prompt).toBeUndefined();
|
|
213
|
+
expect(event!.payload.result).toBeUndefined();
|
|
214
|
+
});
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
describe("Tool Calls", () => {
|
|
218
|
+
// Tool calls are tested in NodeLLMSpanProcessor.test.ts with mocks
|
|
219
|
+
// These tests verify tool calls work with MemoryAdapter
|
|
220
|
+
it("should process tool call spans", async () => {
|
|
221
|
+
// Use mock store since tool calls don't have model info for TraceSummary
|
|
222
|
+
const mockStore = {
|
|
223
|
+
saveEvent: vi.fn().mockResolvedValue(undefined)
|
|
224
|
+
} as any;
|
|
225
|
+
const toolProcessor = new NodeLLMSpanProcessor(mockStore);
|
|
226
|
+
|
|
227
|
+
toolProcessor.onEnd(toolCallWeather);
|
|
228
|
+
await toolProcessor.forceFlush();
|
|
229
|
+
|
|
230
|
+
expect(mockStore.saveEvent).toHaveBeenCalled();
|
|
231
|
+
const event = mockStore.saveEvent.mock.calls[0][0];
|
|
232
|
+
expect(event.eventType).toBe("tool.end");
|
|
233
|
+
expect(event.payload.tool.name).toBe("getWeather");
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
it("should capture tool args and result", async () => {
|
|
237
|
+
const mockStore = {
|
|
238
|
+
saveEvent: vi.fn().mockResolvedValue(undefined)
|
|
239
|
+
} as any;
|
|
240
|
+
const toolProcessor = new NodeLLMSpanProcessor(mockStore);
|
|
241
|
+
|
|
242
|
+
toolProcessor.onEnd(toolCallWeather);
|
|
243
|
+
await toolProcessor.forceFlush();
|
|
244
|
+
|
|
245
|
+
const event = mockStore.saveEvent.mock.calls[0][0];
|
|
246
|
+
expect(event.payload.tool).toEqual({
|
|
247
|
+
name: "getWeather",
|
|
248
|
+
id: "call_abc123",
|
|
249
|
+
args: { city: "London", units: "celsius" },
|
|
250
|
+
result: { temperature: 18, condition: "cloudy" }
|
|
251
|
+
});
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
it("should handle multiple tool calls", async () => {
|
|
255
|
+
const mockStore = {
|
|
256
|
+
saveEvent: vi.fn().mockResolvedValue(undefined)
|
|
257
|
+
} as any;
|
|
258
|
+
const toolProcessor = new NodeLLMSpanProcessor(mockStore);
|
|
259
|
+
|
|
260
|
+
toolProcessor.onEnd(toolCallWeather);
|
|
261
|
+
toolProcessor.onEnd(toolCallDatabase);
|
|
262
|
+
await toolProcessor.forceFlush();
|
|
263
|
+
|
|
264
|
+
expect(mockStore.saveEvent).toHaveBeenCalledTimes(2);
|
|
265
|
+
});
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
describe("Error Handling", () => {
|
|
269
|
+
it("should capture error spans with request.error event type", async () => {
|
|
270
|
+
processor.onEnd(errorRateLimit);
|
|
271
|
+
await processor.forceFlush();
|
|
272
|
+
|
|
273
|
+
const event = await getFirstEvent(store);
|
|
274
|
+
expect(event).toBeDefined();
|
|
275
|
+
expect(event!.eventType).toBe("request.error");
|
|
276
|
+
expect(event!.payload.error).toBe("Rate limit exceeded. Please retry after 60 seconds.");
|
|
277
|
+
});
|
|
278
|
+
|
|
279
|
+
it("should capture context length errors", async () => {
|
|
280
|
+
processor.onEnd(errorContextLength);
|
|
281
|
+
await processor.forceFlush();
|
|
282
|
+
|
|
283
|
+
const event = await getFirstEvent(store);
|
|
284
|
+
expect(event).toBeDefined();
|
|
285
|
+
expect(event!.eventType).toBe("request.error");
|
|
286
|
+
expect(event!.payload.error).toContain("maximum context length");
|
|
287
|
+
});
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
describe("Span Filtering", () => {
|
|
291
|
+
it("should filter out internal .doGenerate spans", async () => {
|
|
292
|
+
processor.onEnd(internalDoGenerate);
|
|
293
|
+
await processor.forceFlush();
|
|
294
|
+
|
|
295
|
+
const traces = await getTraceSummaries(store);
|
|
296
|
+
expect(traces).toHaveLength(0);
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
it("should filter out HTTP spans", async () => {
|
|
300
|
+
processor.onEnd(httpRequest);
|
|
301
|
+
await processor.forceFlush();
|
|
302
|
+
|
|
303
|
+
const traces = await getTraceSummaries(store);
|
|
304
|
+
expect(traces).toHaveLength(0);
|
|
305
|
+
});
|
|
306
|
+
|
|
307
|
+
it("should process all valid spans and filter out invalid ones", async () => {
|
|
308
|
+
// Process all spans
|
|
309
|
+
for (const span of allValidSpans) {
|
|
310
|
+
processor.onEnd(span);
|
|
311
|
+
}
|
|
312
|
+
for (const span of filteredOutSpans) {
|
|
313
|
+
processor.onEnd(span);
|
|
314
|
+
}
|
|
315
|
+
await processor.forceFlush();
|
|
316
|
+
|
|
317
|
+
const traces = await getTraceSummaries(store);
|
|
318
|
+
expect(traces).toHaveLength(allValidSpans.length);
|
|
319
|
+
});
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
describe("OTel Trace Context", () => {
|
|
323
|
+
it("should preserve OTel trace and span IDs", async () => {
|
|
324
|
+
processor.onEnd(openaiGenerateText);
|
|
325
|
+
await processor.forceFlush();
|
|
326
|
+
|
|
327
|
+
const event = await getFirstEvent(store);
|
|
328
|
+
expect(event).toBeDefined();
|
|
329
|
+
expect(event!.payload.otel).toEqual({
|
|
330
|
+
traceId: "openai-trace-001",
|
|
331
|
+
spanId: "openai-span-001",
|
|
332
|
+
parentSpanId: undefined
|
|
333
|
+
});
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
it("should include parent span ID for child spans", async () => {
|
|
337
|
+
// Use mock store for tool call which has parent span
|
|
338
|
+
const mockStore = {
|
|
339
|
+
saveEvent: vi.fn().mockResolvedValue(undefined)
|
|
340
|
+
} as any;
|
|
341
|
+
const toolProcessor = new NodeLLMSpanProcessor(mockStore);
|
|
342
|
+
|
|
343
|
+
toolProcessor.onEnd(toolCallWeather);
|
|
344
|
+
await toolProcessor.forceFlush();
|
|
345
|
+
|
|
346
|
+
const event = mockStore.saveEvent.mock.calls[0][0];
|
|
347
|
+
expect(event.payload.otel.parentSpanId).toBe("parent-span-001");
|
|
348
|
+
});
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
describe("Embedding Operations", () => {
|
|
352
|
+
it("should process ai.embed spans", async () => {
|
|
353
|
+
processor.onEnd(openaiEmbed);
|
|
354
|
+
await processor.forceFlush();
|
|
355
|
+
|
|
356
|
+
const traces = await getTraceSummaries(store);
|
|
357
|
+
expect(traces[0].model).toBe("text-embedding-3-small");
|
|
358
|
+
|
|
359
|
+
const event = await getFirstEvent(store);
|
|
360
|
+
expect(event!.payload.operationType).toBe("ai.embed");
|
|
361
|
+
});
|
|
362
|
+
|
|
363
|
+
it("should process ai.embedMany spans", async () => {
|
|
364
|
+
processor.onEnd(openaiEmbedMany);
|
|
365
|
+
await processor.forceFlush();
|
|
366
|
+
|
|
367
|
+
const traces = await getTraceSummaries(store);
|
|
368
|
+
expect(traces[0].model).toBe("text-embedding-3-large");
|
|
369
|
+
|
|
370
|
+
const event = await getFirstEvent(store);
|
|
371
|
+
expect(event!.payload.operationType).toBe("ai.embedMany");
|
|
372
|
+
});
|
|
373
|
+
});
|
|
374
|
+
|
|
375
|
+
describe("Cost Estimation", () => {
|
|
376
|
+
it("should estimate cost for OpenAI gpt-4o-mini", async () => {
|
|
377
|
+
processor.onEnd(openaiGenerateText);
|
|
378
|
+
await processor.forceFlush();
|
|
379
|
+
|
|
380
|
+
const traces = await getTraceSummaries(store);
|
|
381
|
+
// Cost should be calculated based on token counts
|
|
382
|
+
expect(traces[0].cost).toBeGreaterThan(0);
|
|
383
|
+
expect(typeof traces[0].cost).toBe("number");
|
|
384
|
+
});
|
|
385
|
+
|
|
386
|
+
it("should estimate cost for Anthropic Claude", async () => {
|
|
387
|
+
processor.onEnd(anthropicGenerateText);
|
|
388
|
+
await processor.forceFlush();
|
|
389
|
+
|
|
390
|
+
const traces = await getTraceSummaries(store);
|
|
391
|
+
expect(traces[0].cost).toBeGreaterThan(0);
|
|
392
|
+
});
|
|
393
|
+
});
|
|
394
|
+
|
|
395
|
+
describe("Duration Calculation", () => {
|
|
396
|
+
it("should calculate duration from start/end times", async () => {
|
|
397
|
+
processor.onEnd(openaiGenerateText);
|
|
398
|
+
await processor.forceFlush();
|
|
399
|
+
|
|
400
|
+
const traces = await getTraceSummaries(store);
|
|
401
|
+
// Start: [1706918400, 0], End: [1706918401, 500000000] = 1500ms
|
|
402
|
+
expect(traces[0].duration).toBe(1500);
|
|
403
|
+
});
|
|
404
|
+
|
|
405
|
+
it("should calculate correct duration for streaming spans", async () => {
|
|
406
|
+
processor.onEnd(openaiStreamText);
|
|
407
|
+
await processor.forceFlush();
|
|
408
|
+
|
|
409
|
+
const traces = await getTraceSummaries(store);
|
|
410
|
+
// Start: [1706918500, 0], End: [1706918502, 340000000] = 2340ms
|
|
411
|
+
expect(traces[0].duration).toBe(2340);
|
|
412
|
+
});
|
|
413
|
+
});
|
|
414
|
+
|
|
415
|
+
describe("Concurrent Processing", () => {
|
|
416
|
+
it("should handle many spans concurrently", async () => {
|
|
417
|
+
const spanCount = 100;
|
|
418
|
+
|
|
419
|
+
for (let i = 0; i < spanCount; i++) {
|
|
420
|
+
processor.onEnd({
|
|
421
|
+
...openaiGenerateText,
|
|
422
|
+
spanContext: () => ({ traceId: `trace-${i}`, spanId: `span-${i}` })
|
|
423
|
+
} as any);
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
await processor.forceFlush();
|
|
427
|
+
|
|
428
|
+
const traces = await getTraceSummaries(store, { limit: 200 });
|
|
429
|
+
expect(traces).toHaveLength(spanCount);
|
|
430
|
+
});
|
|
431
|
+
});
|
|
432
|
+
});
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
isAISpan,
|
|
4
|
+
isTopLevelAISpan,
|
|
5
|
+
getOperationType,
|
|
6
|
+
extractAIAttributes,
|
|
7
|
+
hrTimeToDate,
|
|
8
|
+
calculateDurationMs,
|
|
9
|
+
extractProvider,
|
|
10
|
+
extractModel,
|
|
11
|
+
mapStatusToEventType,
|
|
12
|
+
generateRequestId,
|
|
13
|
+
normalizeModelName,
|
|
14
|
+
normalizeProviderName
|
|
15
|
+
} from "../src/utils.js";
|
|
16
|
+
import { OTelReadableSpan } from "../src/types.js";
|
|
17
|
+
|
|
18
|
+
describe("otel utils", () => {
|
|
19
|
+
const mockSpanContext = () => ({
|
|
20
|
+
traceId: "trace-123",
|
|
21
|
+
spanId: "span-456"
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
describe("isAISpan", () => {
|
|
25
|
+
it("should identify Vercel AI spans by name", () => {
|
|
26
|
+
const span = { name: "ai.generateText" } as any;
|
|
27
|
+
expect(isAISpan(span)).toBe(true);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it("should identify spans with ai.operationId attribute", () => {
|
|
31
|
+
const span = {
|
|
32
|
+
name: "custom-operation",
|
|
33
|
+
attributes: { "ai.operationId": "ai.generateText" }
|
|
34
|
+
} as any;
|
|
35
|
+
expect(isAISpan(span)).toBe(true);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
it("should return false for non-AI spans", () => {
|
|
39
|
+
const span = { name: "http.request", attributes: {} } as any;
|
|
40
|
+
expect(isAISpan(span)).toBe(false);
|
|
41
|
+
});
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
describe("isTopLevelAISpan", () => {
|
|
45
|
+
it("should return true for top-level AI spans", () => {
|
|
46
|
+
const span = {
|
|
47
|
+
name: "ai.generateText",
|
|
48
|
+
attributes: {}
|
|
49
|
+
} as any;
|
|
50
|
+
expect(isTopLevelAISpan(span)).toBe(true);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it("should return false for .doGenerate internal spans", () => {
|
|
54
|
+
const span = {
|
|
55
|
+
name: "ai.generateText.doGenerate",
|
|
56
|
+
attributes: {}
|
|
57
|
+
} as any;
|
|
58
|
+
expect(isTopLevelAISpan(span)).toBe(false);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it("should return true for tool calls even if they look like internal spans", () => {
|
|
62
|
+
const span = {
|
|
63
|
+
name: "ai.toolCall",
|
|
64
|
+
attributes: { "ai.operationId": "ai.toolCall" }
|
|
65
|
+
} as any;
|
|
66
|
+
expect(isTopLevelAISpan(span)).toBe(true);
|
|
67
|
+
});
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
describe("extractAIAttributes", () => {
|
|
71
|
+
it("should extract Vercel AI SDK attributes", () => {
|
|
72
|
+
const span = {
|
|
73
|
+
attributes: {
|
|
74
|
+
"ai.model.id": "gpt-4o",
|
|
75
|
+
"ai.model.provider": "openai",
|
|
76
|
+
"ai.usage.promptTokens": 10,
|
|
77
|
+
"ai.usage.completionTokens": 20,
|
|
78
|
+
"ai.prompt": "Hello",
|
|
79
|
+
"ai.response.text": "Hi there"
|
|
80
|
+
}
|
|
81
|
+
} as any;
|
|
82
|
+
|
|
83
|
+
const attrs = extractAIAttributes(span);
|
|
84
|
+
expect(attrs.modelId).toBe("gpt-4o");
|
|
85
|
+
expect(attrs.modelProvider).toBe("openai");
|
|
86
|
+
expect(attrs.promptTokens).toBe(10);
|
|
87
|
+
expect(attrs.completionTokens).toBe(20);
|
|
88
|
+
expect(attrs.prompt).toBe("Hello");
|
|
89
|
+
expect(attrs.responseText).toBe("Hi there");
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it("should extract OTel GenAI standard attributes", () => {
|
|
93
|
+
const span = {
|
|
94
|
+
attributes: {
|
|
95
|
+
"gen_ai.request.model": "claude-3",
|
|
96
|
+
"gen_ai.system": "anthropic",
|
|
97
|
+
"gen_ai.usage.input_tokens": 15,
|
|
98
|
+
"gen_ai.usage.output_tokens": 25
|
|
99
|
+
}
|
|
100
|
+
} as any;
|
|
101
|
+
|
|
102
|
+
const attrs = extractAIAttributes(span);
|
|
103
|
+
expect(attrs.genAiRequestModel).toBe("claude-3");
|
|
104
|
+
expect(attrs.genAiSystem).toBe("anthropic");
|
|
105
|
+
expect(attrs.genAiInputTokens).toBe(15);
|
|
106
|
+
expect(attrs.genAiOutputTokens).toBe(25);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it("should extract custom metadata", () => {
|
|
110
|
+
const span = {
|
|
111
|
+
attributes: {
|
|
112
|
+
"ai.telemetry.metadata.userId": "user-1",
|
|
113
|
+
"ai.telemetry.metadata.env": "prod"
|
|
114
|
+
}
|
|
115
|
+
} as any;
|
|
116
|
+
|
|
117
|
+
const attrs = extractAIAttributes(span);
|
|
118
|
+
expect(attrs.metadata).toEqual({
|
|
119
|
+
userId: "user-1",
|
|
120
|
+
env: "prod"
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
describe("hrTimeToDate", () => {
|
|
126
|
+
it("should convert OTel hrTime to Date", () => {
|
|
127
|
+
const now = Date.now();
|
|
128
|
+
const seconds = Math.floor(now / 1000);
|
|
129
|
+
const nanos = (now % 1000) * 1_000_000;
|
|
130
|
+
|
|
131
|
+
const date = hrTimeToDate([seconds, nanos]);
|
|
132
|
+
expect(date.getTime()).toBe(now);
|
|
133
|
+
});
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
describe("calculateDurationMs", () => {
|
|
137
|
+
it("should calculate duration between start and end times", () => {
|
|
138
|
+
const start: [number, number] = [1000, 0];
|
|
139
|
+
const end: [number, number] = [1001, 500_000_000]; // 1.5 seconds later
|
|
140
|
+
|
|
141
|
+
expect(calculateDurationMs(start, end)).toBe(1500);
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
describe("extractProvider", () => {
|
|
146
|
+
it("should infer provider from model ID if not explicitly provided", () => {
|
|
147
|
+
expect(extractProvider({ modelId: "gpt-4o" })).toBe("openai");
|
|
148
|
+
expect(extractProvider({ modelId: "claude-3" })).toBe("anthropic");
|
|
149
|
+
expect(extractProvider({ modelId: "gemini-pro" })).toBe("google");
|
|
150
|
+
expect(extractProvider({ modelId: "deepseek-chat" })).toBe("deepseek");
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it("should use explicit provider if available", () => {
|
|
154
|
+
expect(extractProvider({ modelProvider: "custom-provider", modelId: "gpt-4" })).toBe(
|
|
155
|
+
"custom-provider"
|
|
156
|
+
);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it("should normalize Vercel AI SDK provider formats", () => {
|
|
160
|
+
expect(extractProvider({ modelProvider: "openai.responses" })).toBe("openai");
|
|
161
|
+
expect(extractProvider({ modelProvider: "anthropic.messages" })).toBe("anthropic");
|
|
162
|
+
expect(extractProvider({ modelProvider: "google.generativeai" })).toBe("google");
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
it("should normalize genAiSystem provider formats", () => {
|
|
166
|
+
expect(extractProvider({ genAiSystem: "openai.chat" })).toBe("openai");
|
|
167
|
+
expect(extractProvider({ genAiSystem: "anthropic.messages" })).toBe("anthropic");
|
|
168
|
+
});
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
describe("normalizeProviderName", () => {
|
|
172
|
+
it("should strip operation suffixes from provider names", () => {
|
|
173
|
+
expect(normalizeProviderName("openai.responses")).toBe("openai");
|
|
174
|
+
expect(normalizeProviderName("openai.chat")).toBe("openai");
|
|
175
|
+
expect(normalizeProviderName("anthropic.messages")).toBe("anthropic");
|
|
176
|
+
expect(normalizeProviderName("google.generativeai")).toBe("google");
|
|
177
|
+
expect(normalizeProviderName("google.vertex")).toBe("google");
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
it("should pass through standard provider names unchanged", () => {
|
|
181
|
+
expect(normalizeProviderName("openai")).toBe("openai");
|
|
182
|
+
expect(normalizeProviderName("anthropic")).toBe("anthropic");
|
|
183
|
+
expect(normalizeProviderName("google")).toBe("google");
|
|
184
|
+
expect(normalizeProviderName("custom-provider")).toBe("custom-provider");
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
it("should handle edge cases", () => {
|
|
188
|
+
expect(normalizeProviderName("")).toBe("");
|
|
189
|
+
expect(normalizeProviderName("unknown")).toBe("unknown");
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
describe("mapStatusToEventType", () => {
|
|
194
|
+
it("should map OTel status to NodeLLM event types", () => {
|
|
195
|
+
expect(mapStatusToEventType(1, "ai.generateText")).toBe("request.end");
|
|
196
|
+
expect(mapStatusToEventType(2, "ai.generateText")).toBe("request.error");
|
|
197
|
+
expect(mapStatusToEventType(1, "ai.toolCall")).toBe("tool.end");
|
|
198
|
+
expect(mapStatusToEventType(2, "ai.toolCall")).toBe("tool.error");
|
|
199
|
+
});
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
describe("generateRequestId", () => {
|
|
203
|
+
it("should generate a request ID from trace and span context", () => {
|
|
204
|
+
const span = {
|
|
205
|
+
spanContext: () => ({
|
|
206
|
+
traceId: "00112233445566778899aabbccddeeff",
|
|
207
|
+
spanId: "1122334455667788"
|
|
208
|
+
})
|
|
209
|
+
} as any;
|
|
210
|
+
expect(generateRequestId(span)).toBe("otel_00112233_11223344");
|
|
211
|
+
});
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
describe("normalizeModelName", () => {
|
|
215
|
+
it("should normalize Vercel AI SDK openai.responses format", () => {
|
|
216
|
+
expect(normalizeModelName("openai.responses/gpt-4o-mini")).toBe("gpt-4o-mini");
|
|
217
|
+
expect(normalizeModelName("openai.responses/gpt-4o")).toBe("gpt-4o");
|
|
218
|
+
expect(normalizeModelName("openai.chat/gpt-3.5-turbo")).toBe("gpt-3.5-turbo");
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
it("should normalize Vercel AI SDK anthropic format", () => {
|
|
222
|
+
expect(normalizeModelName("anthropic.messages/claude-3-5-sonnet-20241022")).toBe(
|
|
223
|
+
"claude-3-5-sonnet-20241022"
|
|
224
|
+
);
|
|
225
|
+
expect(normalizeModelName("anthropic.messages/claude-3-opus")).toBe("claude-3-opus");
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
it("should normalize Vercel AI SDK google format", () => {
|
|
229
|
+
expect(normalizeModelName("google.generativeai/gemini-1.5-pro")).toBe("gemini-1.5-pro");
|
|
230
|
+
expect(normalizeModelName("google.vertex/gemini-pro")).toBe("gemini-pro");
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
it("should pass through standard model names unchanged", () => {
|
|
234
|
+
expect(normalizeModelName("gpt-4o-mini")).toBe("gpt-4o-mini");
|
|
235
|
+
expect(normalizeModelName("claude-3-5-sonnet")).toBe("claude-3-5-sonnet");
|
|
236
|
+
expect(normalizeModelName("gemini-pro")).toBe("gemini-pro");
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
it("should handle edge cases", () => {
|
|
240
|
+
expect(normalizeModelName("unknown")).toBe("unknown");
|
|
241
|
+
expect(normalizeModelName("")).toBe("");
|
|
242
|
+
expect(normalizeModelName("provider/")).toBe("provider/"); // returns original when last part is empty
|
|
243
|
+
});
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
describe("extractModel", () => {
|
|
247
|
+
it("should extract and normalize model from response model", () => {
|
|
248
|
+
expect(extractModel({ responseModel: "openai.responses/gpt-4o-mini" })).toBe("gpt-4o-mini");
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
it("should extract and normalize model from modelId", () => {
|
|
252
|
+
expect(extractModel({ modelId: "anthropic.messages/claude-3-5-sonnet" })).toBe(
|
|
253
|
+
"claude-3-5-sonnet"
|
|
254
|
+
);
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
it("should use genAi response model as fallback", () => {
|
|
258
|
+
expect(extractModel({ genAiResponseModel: "gpt-4" })).toBe("gpt-4");
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
it("should use genAi request model as fallback", () => {
|
|
262
|
+
expect(extractModel({ genAiRequestModel: "claude-3" })).toBe("claude-3");
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
it("should return unknown when no model is present", () => {
|
|
266
|
+
expect(extractModel({})).toBe("unknown");
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
it("should prefer responseModel over modelId", () => {
|
|
270
|
+
expect(
|
|
271
|
+
extractModel({
|
|
272
|
+
responseModel: "openai.responses/gpt-4o",
|
|
273
|
+
modelId: "gpt-3.5-turbo"
|
|
274
|
+
})
|
|
275
|
+
).toBe("gpt-4o");
|
|
276
|
+
});
|
|
277
|
+
});
|
|
278
|
+
});
|