autotel 2.26.0 → 2.26.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attribute-redacting-processor.cjs +14 -6
- package/dist/attribute-redacting-processor.d.cts +63 -1
- package/dist/attribute-redacting-processor.d.ts +63 -1
- package/dist/attribute-redacting-processor.js +1 -1
- package/dist/attributes.cjs +21 -21
- package/dist/attributes.js +2 -2
- package/dist/auto.cjs +8 -8
- package/dist/auto.js +6 -6
- package/dist/{chunk-RUD7KS4R.js → chunk-3SDILILG.js} +3 -3
- package/dist/{chunk-RUD7KS4R.js.map → chunk-3SDILILG.js.map} +1 -1
- package/dist/{chunk-B33XPEKY.js → chunk-55ER2KD5.js} +4 -4
- package/dist/chunk-55ER2KD5.js.map +1 -0
- package/dist/{chunk-UJJPTSEI.cjs → chunk-563EL6O6.cjs} +81 -14
- package/dist/chunk-563EL6O6.cjs.map +1 -0
- package/dist/{chunk-TS7IHIRW.cjs → chunk-6YGUN7IY.cjs} +5 -5
- package/dist/{chunk-TS7IHIRW.cjs.map → chunk-6YGUN7IY.cjs.map} +1 -1
- package/dist/{chunk-XDKK53OL.js → chunk-A4E5AQFK.js} +3 -3
- package/dist/{chunk-XDKK53OL.js.map → chunk-A4E5AQFK.js.map} +1 -1
- package/dist/{chunk-WAB4CHBU.js → chunk-BJ2XPN77.js} +3 -3
- package/dist/{chunk-WAB4CHBU.js.map → chunk-BJ2XPN77.js.map} +1 -1
- package/dist/{chunk-KZEC4CHV.cjs → chunk-CEAQK2QY.cjs} +5 -5
- package/dist/{chunk-KZEC4CHV.cjs.map → chunk-CEAQK2QY.cjs.map} +1 -1
- package/dist/chunk-CMNGGTQL.cjs +349 -0
- package/dist/chunk-CMNGGTQL.cjs.map +1 -0
- package/dist/{chunk-VYA6QDNA.js → chunk-DPSA4QLA.js} +4 -2
- package/dist/chunk-DPSA4QLA.js.map +1 -0
- package/dist/{chunk-M4US3P4K.js → chunk-ER43K7ES.js} +3 -3
- package/dist/{chunk-M4US3P4K.js.map → chunk-ER43K7ES.js.map} +1 -1
- package/dist/{chunk-AZ24DJAG.cjs → chunk-FU6R566Y.cjs} +4 -4
- package/dist/chunk-FU6R566Y.cjs.map +1 -0
- package/dist/{chunk-4PTCDOZY.js → chunk-HPUGKUMZ.js} +4 -4
- package/dist/{chunk-4PTCDOZY.js.map → chunk-HPUGKUMZ.js.map} +1 -1
- package/dist/{chunk-XRBP4RYL.cjs → chunk-JKIMEPI2.cjs} +4 -4
- package/dist/{chunk-XRBP4RYL.cjs.map → chunk-JKIMEPI2.cjs.map} +1 -1
- package/dist/{chunk-N344PVE5.cjs → chunk-OBWXM4NN.cjs} +9 -9
- package/dist/{chunk-N344PVE5.cjs.map → chunk-OBWXM4NN.cjs.map} +1 -1
- package/dist/{chunk-OFPZULMQ.cjs → chunk-OC6X2VIN.cjs} +8 -8
- package/dist/{chunk-OFPZULMQ.cjs.map → chunk-OC6X2VIN.cjs.map} +1 -1
- package/dist/{chunk-GTD3NXOS.js → chunk-QC5MNKVF.js} +4 -4
- package/dist/{chunk-GTD3NXOS.js.map → chunk-QC5MNKVF.js.map} +1 -1
- package/dist/chunk-TDNKIHKT.js +341 -0
- package/dist/chunk-TDNKIHKT.js.map +1 -0
- package/dist/{chunk-DGPUZ6TE.js → chunk-U54FTVFH.js} +3 -3
- package/dist/{chunk-DGPUZ6TE.js.map → chunk-U54FTVFH.js.map} +1 -1
- package/dist/{chunk-ZJ5GXCOT.cjs → chunk-UTZR7P7E.cjs} +36 -36
- package/dist/{chunk-ZJ5GXCOT.cjs.map → chunk-UTZR7P7E.cjs.map} +1 -1
- package/dist/{chunk-7FIGORWI.cjs → chunk-VH77IPJN.cjs} +4 -2
- package/dist/chunk-VH77IPJN.cjs.map +1 -0
- package/dist/{chunk-EXOXDI5A.js → chunk-W35FVJBC.js} +73 -8
- package/dist/chunk-W35FVJBC.js.map +1 -0
- package/dist/{chunk-II7GFVAF.cjs → chunk-WZOKY3PW.cjs} +13 -13
- package/dist/{chunk-II7GFVAF.cjs.map → chunk-WZOKY3PW.cjs.map} +1 -1
- package/dist/{chunk-CMADDTHY.cjs → chunk-YEVCD6DR.cjs} +7 -7
- package/dist/{chunk-CMADDTHY.cjs.map → chunk-YEVCD6DR.cjs.map} +1 -1
- package/dist/{chunk-RXFZKLRQ.js → chunk-YN7USLHW.js} +3 -3
- package/dist/{chunk-RXFZKLRQ.js.map → chunk-YN7USLHW.js.map} +1 -1
- package/dist/decorators.cjs +7 -7
- package/dist/decorators.js +7 -7
- package/dist/event.cjs +10 -10
- package/dist/event.js +7 -7
- package/dist/functional.cjs +14 -14
- package/dist/functional.js +7 -7
- package/dist/index.cjs +340 -97
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +205 -3
- package/dist/index.d.ts +205 -3
- package/dist/index.js +257 -33
- package/dist/index.js.map +1 -1
- package/dist/{init-QSj7X6zU.d.cts → init-CMuTaFAV.d.cts} +26 -1
- package/dist/{init-FiR_glVc.d.ts → init-D6JfWEjL.d.ts} +26 -1
- package/dist/instrumentation.cjs +14 -14
- package/dist/instrumentation.js +6 -6
- package/dist/logger.cjs +8 -8
- package/dist/logger.js +1 -1
- package/dist/messaging.cjs +11 -11
- package/dist/messaging.js +8 -8
- package/dist/metric.cjs +1 -1
- package/dist/metric.js +1 -1
- package/dist/sampling.cjs +15 -15
- package/dist/sampling.js +2 -2
- package/dist/semantic-helpers.cjs +12 -12
- package/dist/semantic-helpers.js +8 -8
- package/dist/tail-sampling-processor.cjs +4 -4
- package/dist/tail-sampling-processor.js +3 -3
- package/dist/testing.cjs +1 -1
- package/dist/testing.js +1 -1
- package/dist/webhook.cjs +9 -8
- package/dist/webhook.cjs.map +1 -1
- package/dist/webhook.js +8 -7
- package/dist/webhook.js.map +1 -1
- package/dist/workflow-distributed.cjs +9 -9
- package/dist/workflow-distributed.js +7 -7
- package/dist/workflow.cjs +12 -12
- package/dist/workflow.js +8 -8
- package/dist/yaml-config.cjs +6 -6
- package/dist/yaml-config.d.cts +1 -1
- package/dist/yaml-config.d.ts +1 -1
- package/dist/yaml-config.js +3 -3
- package/package.json +1 -1
- package/src/attribute-redacting-processor.test.ts +81 -16
- package/src/attribute-redacting-processor.ts +278 -24
- package/src/autotel-logger.ts +2 -2
- package/src/gen-ai-events.test.ts +135 -0
- package/src/gen-ai-events.ts +199 -0
- package/src/gen-ai-metrics.test.ts +96 -0
- package/src/gen-ai-metrics.ts +128 -0
- package/src/index.ts +28 -1
- package/src/init.ts +117 -2
- package/src/request-logger.test.ts +266 -1
- package/src/request-logger.ts +115 -16
- package/src/structured-error.ts +54 -1
- package/dist/chunk-7FIGORWI.cjs.map +0 -1
- package/dist/chunk-AZ24DJAG.cjs.map +0 -1
- package/dist/chunk-B33XPEKY.js.map +0 -1
- package/dist/chunk-ELW34S4C.cjs +0 -173
- package/dist/chunk-ELW34S4C.cjs.map +0 -1
- package/dist/chunk-EXOXDI5A.js.map +0 -1
- package/dist/chunk-SNINLBEE.js +0 -167
- package/dist/chunk-SNINLBEE.js.map +0 -1
- package/dist/chunk-UJJPTSEI.cjs.map +0 -1
- package/dist/chunk-VYA6QDNA.js.map +0 -1
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Span event helpers for LLM lifecycle, aligned with the OpenTelemetry
|
|
3
|
+
* GenAI semantic conventions.
|
|
4
|
+
*
|
|
5
|
+
* Span events are timestamped points within a span — they render as dots
|
|
6
|
+
* on the trace timeline in Jaeger / Tempo / Langfuse / Arize. Use them
|
|
7
|
+
* to mark lifecycle moments the span attributes alone can't express:
|
|
8
|
+
*
|
|
9
|
+
* - When the prompt was sent (vs. when the first token arrived)
|
|
10
|
+
* - When each retry attempt started, and why
|
|
11
|
+
* - When a streaming response produced its first token (TTFT)
|
|
12
|
+
* - When a tool was invoked
|
|
13
|
+
*
|
|
14
|
+
* Every helper pins the event name + attribute keys to the published
|
|
15
|
+
* spec so downstream tooling (autotel-mcp, Langfuse, vendor UIs) can
|
|
16
|
+
* render them consistently.
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```typescript
|
|
20
|
+
* import { trace, recordPromptSent, recordResponseReceived, recordRetry } from 'autotel';
|
|
21
|
+
*
|
|
22
|
+
* export const chat = trace('chat', ctx => async (prompt: string) => {
|
|
23
|
+
* recordPromptSent(ctx, { model: 'gpt-4o', messageCount: 1 });
|
|
24
|
+
*
|
|
25
|
+
* for (let attempt = 1; attempt <= 3; attempt++) {
|
|
26
|
+
* try {
|
|
27
|
+
* const res = await openai.chat.completions.create({...});
|
|
28
|
+
* recordResponseReceived(ctx, {
|
|
29
|
+
* model: res.model,
|
|
30
|
+
* promptTokens: res.usage?.prompt_tokens,
|
|
31
|
+
* completionTokens: res.usage?.completion_tokens,
|
|
32
|
+
* finishReasons: res.choices.map(c => c.finish_reason),
|
|
33
|
+
* });
|
|
34
|
+
* return res;
|
|
35
|
+
* } catch (err) {
|
|
36
|
+
* recordRetry(ctx, { attempt, reason: 'rate_limit', delayMs: 500 });
|
|
37
|
+
* await sleep(500 * attempt);
|
|
38
|
+
* }
|
|
39
|
+
* }
|
|
40
|
+
* });
|
|
41
|
+
* ```
|
|
42
|
+
*/
|
|
43
|
+
|
|
44
|
+
import type { TraceContext } from './trace-context';
|
|
45
|
+
|
|
46
|
+
type EventAttrs = Record<string, string | number | boolean>;
|
|
47
|
+
|
|
48
|
+
/** Attributes expected on a `gen_ai.prompt.sent` event. */
|
|
49
|
+
export interface PromptSentEvent {
|
|
50
|
+
/** Model the caller intends to invoke (may differ from response model). */
|
|
51
|
+
model?: string;
|
|
52
|
+
/** Estimated input token count, when known before the call. */
|
|
53
|
+
promptTokens?: number;
|
|
54
|
+
/** Number of messages in a chat request (system + user + assistant). */
|
|
55
|
+
messageCount?: number;
|
|
56
|
+
/** Free-form operation kind — `chat` / `completion` / `embedding`. */
|
|
57
|
+
operation?: string;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Attributes expected on a `gen_ai.response.received` event. */
|
|
61
|
+
export interface ResponseReceivedEvent {
|
|
62
|
+
/** Model the provider actually served (may be more specific than requested). */
|
|
63
|
+
model?: string;
|
|
64
|
+
promptTokens?: number;
|
|
65
|
+
completionTokens?: number;
|
|
66
|
+
totalTokens?: number;
|
|
67
|
+
/** `stop`, `length`, `content_filter`, `tool_calls`, etc. */
|
|
68
|
+
finishReasons?: string[];
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Attributes expected on a `gen_ai.retry` event. */
|
|
72
|
+
export interface RetryEvent {
|
|
73
|
+
attempt: number;
|
|
74
|
+
/** `rate_limit` | `timeout` | `provider_error` | custom label. */
|
|
75
|
+
reason?: string;
|
|
76
|
+
/** How long we'll wait before the next attempt. */
|
|
77
|
+
delayMs?: number;
|
|
78
|
+
/** HTTP status that triggered the retry, when applicable. */
|
|
79
|
+
statusCode?: number;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Attributes expected on a `gen_ai.tool.call` event. */
|
|
83
|
+
export interface ToolCallEvent {
|
|
84
|
+
toolName: string;
|
|
85
|
+
/** Call identifier so responses can be correlated back to calls. */
|
|
86
|
+
toolCallId?: string;
|
|
87
|
+
/** Pre-serialised tool arguments; omit if sensitive. */
|
|
88
|
+
arguments?: string;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Attributes expected on a `gen_ai.stream.first_token` event. */
|
|
92
|
+
export interface StreamFirstTokenEvent {
|
|
93
|
+
/** Tokens streamed so far, if the caller tracks that. */
|
|
94
|
+
tokensSoFar?: number;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Record that a prompt was dispatched to the provider. Typically called
|
|
99
|
+
* before `await provider.chat.completions.create(...)`.
|
|
100
|
+
*/
|
|
101
|
+
export function recordPromptSent(
|
|
102
|
+
ctx: TraceContext,
|
|
103
|
+
event: PromptSentEvent = {},
|
|
104
|
+
): void {
|
|
105
|
+
ctx.addEvent('gen_ai.prompt.sent', buildPromptSentAttrs(event));
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Record a successful provider response. Call after the response arrives
|
|
110
|
+
* (for non-streaming) or after the stream completes.
|
|
111
|
+
*/
|
|
112
|
+
export function recordResponseReceived(
|
|
113
|
+
ctx: TraceContext,
|
|
114
|
+
event: ResponseReceivedEvent = {},
|
|
115
|
+
): void {
|
|
116
|
+
ctx.addEvent('gen_ai.response.received', buildResponseAttrs(event));
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Record a retry attempt on an LLM call. Call *before* sleeping for
|
|
121
|
+
* `delayMs` so the event timestamp accurately marks when the retry
|
|
122
|
+
* decision was made.
|
|
123
|
+
*/
|
|
124
|
+
export function recordRetry(ctx: TraceContext, event: RetryEvent): void {
|
|
125
|
+
ctx.addEvent('gen_ai.retry', buildRetryAttrs(event));
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Record a tool / function call made in the course of an agent step.
|
|
130
|
+
* Emits an event rather than a child span because many frameworks fire
|
|
131
|
+
* several tool calls within a single provider response.
|
|
132
|
+
*/
|
|
133
|
+
export function recordToolCall(ctx: TraceContext, event: ToolCallEvent): void {
|
|
134
|
+
ctx.addEvent('gen_ai.tool.call', buildToolCallAttrs(event));
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Record the time-to-first-token for a streaming response. Pair with
|
|
139
|
+
* `recordResponseReceived` at the end so the span carries both the TTFT
|
|
140
|
+
* marker and the final usage numbers.
|
|
141
|
+
*/
|
|
142
|
+
export function recordStreamFirstToken(
|
|
143
|
+
ctx: TraceContext,
|
|
144
|
+
event: StreamFirstTokenEvent = {},
|
|
145
|
+
): void {
|
|
146
|
+
ctx.addEvent('gen_ai.stream.first_token', buildStreamFirstTokenAttrs(event));
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// ---- Attribute builders -------------------------------------------------
|
|
150
|
+
|
|
151
|
+
function buildPromptSentAttrs(event: PromptSentEvent): EventAttrs {
|
|
152
|
+
const attrs: EventAttrs = {};
|
|
153
|
+
if (event.model) attrs['gen_ai.request.model'] = event.model;
|
|
154
|
+
if (event.promptTokens !== undefined)
|
|
155
|
+
attrs['gen_ai.usage.input_tokens'] = event.promptTokens;
|
|
156
|
+
if (event.messageCount !== undefined)
|
|
157
|
+
attrs['gen_ai.request.message_count'] = event.messageCount;
|
|
158
|
+
if (event.operation) attrs['gen_ai.operation.name'] = event.operation;
|
|
159
|
+
return attrs;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function buildResponseAttrs(event: ResponseReceivedEvent): EventAttrs {
|
|
163
|
+
const attrs: EventAttrs = {};
|
|
164
|
+
if (event.model) attrs['gen_ai.response.model'] = event.model;
|
|
165
|
+
if (event.promptTokens !== undefined)
|
|
166
|
+
attrs['gen_ai.usage.input_tokens'] = event.promptTokens;
|
|
167
|
+
if (event.completionTokens !== undefined)
|
|
168
|
+
attrs['gen_ai.usage.output_tokens'] = event.completionTokens;
|
|
169
|
+
if (event.totalTokens !== undefined)
|
|
170
|
+
attrs['gen_ai.usage.total_tokens'] = event.totalTokens;
|
|
171
|
+
if (event.finishReasons && event.finishReasons.length > 0) {
|
|
172
|
+
// Arrays aren't primitive AttributeValues on this context, so join.
|
|
173
|
+
attrs['gen_ai.response.finish_reasons'] = event.finishReasons.join(',');
|
|
174
|
+
}
|
|
175
|
+
return attrs;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function buildRetryAttrs(event: RetryEvent): EventAttrs {
|
|
179
|
+
const attrs: EventAttrs = { 'retry.attempt': event.attempt };
|
|
180
|
+
if (event.reason) attrs['retry.reason'] = event.reason;
|
|
181
|
+
if (event.delayMs !== undefined) attrs['retry.delay_ms'] = event.delayMs;
|
|
182
|
+
if (event.statusCode !== undefined)
|
|
183
|
+
attrs['http.response.status_code'] = event.statusCode;
|
|
184
|
+
return attrs;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function buildToolCallAttrs(event: ToolCallEvent): EventAttrs {
|
|
188
|
+
const attrs: EventAttrs = { 'gen_ai.tool.name': event.toolName };
|
|
189
|
+
if (event.toolCallId) attrs['gen_ai.tool.call.id'] = event.toolCallId;
|
|
190
|
+
if (event.arguments) attrs['gen_ai.tool.arguments'] = event.arguments;
|
|
191
|
+
return attrs;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function buildStreamFirstTokenAttrs(event: StreamFirstTokenEvent): EventAttrs {
|
|
195
|
+
const attrs: EventAttrs = {};
|
|
196
|
+
if (event.tokensSoFar !== undefined)
|
|
197
|
+
attrs['gen_ai.stream.tokens_so_far'] = event.tokensSoFar;
|
|
198
|
+
return attrs;
|
|
199
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { AggregationType } from '@opentelemetry/sdk-metrics';
|
|
3
|
+
import {
|
|
4
|
+
GEN_AI_COST_USD_BUCKETS,
|
|
5
|
+
GEN_AI_DURATION_BUCKETS_SECONDS,
|
|
6
|
+
GEN_AI_TOKEN_USAGE_BUCKETS,
|
|
7
|
+
genAiMetricViews,
|
|
8
|
+
llmHistogramAdvice,
|
|
9
|
+
} from './gen-ai-metrics';
|
|
10
|
+
|
|
11
|
+
describe('gen-ai-metrics', () => {
|
|
12
|
+
it('bucket arrays are strictly ascending (required by Prometheus + OTel)', () => {
|
|
13
|
+
for (const buckets of [
|
|
14
|
+
GEN_AI_DURATION_BUCKETS_SECONDS,
|
|
15
|
+
GEN_AI_TOKEN_USAGE_BUCKETS,
|
|
16
|
+
GEN_AI_COST_USD_BUCKETS,
|
|
17
|
+
]) {
|
|
18
|
+
for (let i = 1; i < buckets.length; i++) {
|
|
19
|
+
expect(
|
|
20
|
+
buckets[i]! > buckets[i - 1]!,
|
|
21
|
+
`index ${i} not ascending: ${buckets[i - 1]} → ${buckets[i]}`,
|
|
22
|
+
).toBe(true);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it('duration buckets cover tail through 5 minutes for reasoning models', () => {
|
|
28
|
+
expect(GEN_AI_DURATION_BUCKETS_SECONDS[0]).toBeLessThanOrEqual(0.05);
|
|
29
|
+
expect(
|
|
30
|
+
GEN_AI_DURATION_BUCKETS_SECONDS[
|
|
31
|
+
GEN_AI_DURATION_BUCKETS_SECONDS.length - 1
|
|
32
|
+
],
|
|
33
|
+
).toBeGreaterThanOrEqual(300);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it('token buckets cover up to a million-token context window', () => {
|
|
37
|
+
expect(
|
|
38
|
+
GEN_AI_TOKEN_USAGE_BUCKETS[GEN_AI_TOKEN_USAGE_BUCKETS.length - 1],
|
|
39
|
+
).toBeGreaterThanOrEqual(1_000_000);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it('cost buckets resolve sub-cent spend', () => {
|
|
43
|
+
expect(GEN_AI_COST_USD_BUCKETS[0]).toBeLessThan(0.001);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it('bucket arrays are frozen — consumers cannot mutate shared state', () => {
|
|
47
|
+
expect(() => {
|
|
48
|
+
(GEN_AI_DURATION_BUCKETS_SECONDS as number[]).push(999);
|
|
49
|
+
}).toThrow();
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it('llmHistogramAdvice returns explicitBucketBoundaries advice shape', () => {
|
|
53
|
+
const advice = llmHistogramAdvice('duration');
|
|
54
|
+
expect(advice.advice.explicitBucketBoundaries).toEqual([
|
|
55
|
+
...GEN_AI_DURATION_BUCKETS_SECONDS,
|
|
56
|
+
]);
|
|
57
|
+
// The returned array is a fresh copy so callers can mutate without
|
|
58
|
+
// affecting the shared constant.
|
|
59
|
+
advice.advice.explicitBucketBoundaries.push(0);
|
|
60
|
+
expect([...GEN_AI_DURATION_BUCKETS_SECONDS]).not.toContain(0);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it('genAiMetricViews targets the OTel GenAI instrument names with the right buckets', () => {
|
|
64
|
+
const views = genAiMetricViews();
|
|
65
|
+
expect(views).toHaveLength(3);
|
|
66
|
+
|
|
67
|
+
const byInstrument = Object.fromEntries(
|
|
68
|
+
views.map((v) => [v.instrumentName, v]),
|
|
69
|
+
);
|
|
70
|
+
expect(
|
|
71
|
+
byInstrument['gen_ai.client.operation.duration']?.aggregation,
|
|
72
|
+
).toEqual({
|
|
73
|
+
type: AggregationType.EXPLICIT_BUCKET_HISTOGRAM,
|
|
74
|
+
options: { boundaries: [...GEN_AI_DURATION_BUCKETS_SECONDS] },
|
|
75
|
+
});
|
|
76
|
+
expect(byInstrument['gen_ai.client.token.usage']?.aggregation).toEqual({
|
|
77
|
+
type: AggregationType.EXPLICIT_BUCKET_HISTOGRAM,
|
|
78
|
+
options: { boundaries: [...GEN_AI_TOKEN_USAGE_BUCKETS] },
|
|
79
|
+
});
|
|
80
|
+
expect(byInstrument['gen_ai.client.cost.usd']?.aggregation).toEqual({
|
|
81
|
+
type: AggregationType.EXPLICIT_BUCKET_HISTOGRAM,
|
|
82
|
+
options: { boundaries: [...GEN_AI_COST_USD_BUCKETS] },
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it('genAiMetricViews accepts extra instruments', () => {
|
|
87
|
+
const views = genAiMetricViews([
|
|
88
|
+
{ instrumentName: 'custom.llm.prompt_tokens', kind: 'tokens' },
|
|
89
|
+
]);
|
|
90
|
+
expect(views).toHaveLength(4);
|
|
91
|
+
const custom = views.find(
|
|
92
|
+
(v) => v.instrumentName === 'custom.llm.prompt_tokens',
|
|
93
|
+
);
|
|
94
|
+
expect(custom).toBeDefined();
|
|
95
|
+
});
|
|
96
|
+
});
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-tuned histogram buckets.
|
|
3
|
+
*
|
|
4
|
+
* Default OpenTelemetry histogram buckets target HTTP latency (0ms–10s)
|
|
5
|
+
* and small counter values. LLM workloads have very different shapes:
|
|
6
|
+
*
|
|
7
|
+
* - **Duration**: single-token prompts can be fast (50ms), long
|
|
8
|
+
* generations and reasoning models can run for minutes. Default buckets
|
|
9
|
+
* crush everything above 10s into one bucket.
|
|
10
|
+
* - **Token usage**: heavily right-skewed. A single request can range
|
|
11
|
+
* from tens of tokens to the million-token context windows.
|
|
12
|
+
* - **Cost (USD)**: per-request values are tiny (fractions of a cent),
|
|
13
|
+
* so linear buckets waste resolution at the low end.
|
|
14
|
+
*
|
|
15
|
+
* This module exposes empirically-chosen bucket arrays and a View helper
|
|
16
|
+
* so users can apply them to their `MeterProvider` without knowing the
|
|
17
|
+
* exact instrument names emitted by OpenAI/Anthropic/Traceloop plugins.
|
|
18
|
+
*
|
|
19
|
+
* @example
|
|
20
|
+
* ```typescript
|
|
21
|
+
* import { NodeSDK } from '@opentelemetry/sdk-node';
|
|
22
|
+
* import { genAiMetricViews } from 'autotel';
|
|
23
|
+
*
|
|
24
|
+
* const sdk = new NodeSDK({
|
|
25
|
+
* serviceName: 'my-agent',
|
|
26
|
+
* views: [...genAiMetricViews()],
|
|
27
|
+
* });
|
|
28
|
+
* sdk.start();
|
|
29
|
+
* ```
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import { AggregationType, type ViewOptions } from '@opentelemetry/sdk-metrics';
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Duration buckets for LLM operations, in **seconds**. Covers fast
|
|
36
|
+
* completions (50ms) through long-running reasoning jobs (5 min).
|
|
37
|
+
*
|
|
38
|
+
* Aligns with the OTel GenAI semantic conventions' published advice for
|
|
39
|
+
* `gen_ai.client.operation.duration`.
|
|
40
|
+
*/
|
|
41
|
+
export const GEN_AI_DURATION_BUCKETS_SECONDS: readonly number[] = Object.freeze(
|
|
42
|
+
[0.01, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 30, 60, 120, 300],
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Token-count buckets for prompt, completion, and total token histograms.
|
|
47
|
+
* Ranges from tiny prompts to million-token context windows.
|
|
48
|
+
*
|
|
49
|
+
* Aligns with the OTel GenAI semantic conventions' published advice for
|
|
50
|
+
* `gen_ai.client.token.usage`.
|
|
51
|
+
*/
|
|
52
|
+
export const GEN_AI_TOKEN_USAGE_BUCKETS: readonly number[] = Object.freeze([
|
|
53
|
+
1, 4, 16, 64, 256, 1_024, 4_096, 16_384, 65_536, 262_144, 1_048_576,
|
|
54
|
+
4_194_304,
|
|
55
|
+
]);
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* USD cost buckets. Sub-cent resolution at the low end (fractions of a
|
|
59
|
+
* cent per small call) up to tens of dollars (batch jobs, Opus/o1 runs).
|
|
60
|
+
*/
|
|
61
|
+
export const GEN_AI_COST_USD_BUCKETS: readonly number[] = Object.freeze([
|
|
62
|
+
0.000_01, 0.000_1, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 50,
|
|
63
|
+
]);
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Instrument-level advice object for `createHistogram(name, advice)`.
|
|
67
|
+
* Use when you control the instrument creation (e.g. custom business
|
|
68
|
+
* LLM metrics); `genAiMetricViews()` is better when the metric comes
|
|
69
|
+
* from a third-party plugin.
|
|
70
|
+
*/
|
|
71
|
+
export function llmHistogramAdvice(kind: 'duration' | 'tokens' | 'cost'): {
|
|
72
|
+
advice: { explicitBucketBoundaries: number[] };
|
|
73
|
+
} {
|
|
74
|
+
const boundaries =
|
|
75
|
+
kind === 'duration'
|
|
76
|
+
? GEN_AI_DURATION_BUCKETS_SECONDS
|
|
77
|
+
: kind === 'tokens'
|
|
78
|
+
? GEN_AI_TOKEN_USAGE_BUCKETS
|
|
79
|
+
: GEN_AI_COST_USD_BUCKETS;
|
|
80
|
+
return { advice: { explicitBucketBoundaries: [...boundaries] } };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Returns `View`s that re-bucket the standard OTel GenAI histograms. Pass
|
|
85
|
+
* the result to your `MeterProvider`'s `views` option.
|
|
86
|
+
*
|
|
87
|
+
* Matches instrument names emitted by:
|
|
88
|
+
* - OpenTelemetry GenAI autoinstrumentation
|
|
89
|
+
* - OpenInference / OpenLLMetry (traceloop)
|
|
90
|
+
* - Arize Phoenix, LangSmith, etc. that follow the OTel spec
|
|
91
|
+
*
|
|
92
|
+
* Add more instrument patterns via the `extra` argument if you emit
|
|
93
|
+
* custom LLM metrics.
|
|
94
|
+
*/
|
|
95
|
+
export function genAiMetricViews(
|
|
96
|
+
extra: {
|
|
97
|
+
instrumentName: string;
|
|
98
|
+
kind: 'duration' | 'tokens' | 'cost';
|
|
99
|
+
}[] = [],
|
|
100
|
+
): ViewOptions[] {
|
|
101
|
+
const defaults: Array<{
|
|
102
|
+
instrumentName: string;
|
|
103
|
+
kind: 'duration' | 'tokens' | 'cost';
|
|
104
|
+
}> = [
|
|
105
|
+
{ instrumentName: 'gen_ai.client.operation.duration', kind: 'duration' },
|
|
106
|
+
{ instrumentName: 'gen_ai.client.token.usage', kind: 'tokens' },
|
|
107
|
+
// Autotel-emitted cost metric. No-op if you don't emit it.
|
|
108
|
+
{ instrumentName: 'gen_ai.client.cost.usd', kind: 'cost' },
|
|
109
|
+
];
|
|
110
|
+
|
|
111
|
+
return [...defaults, ...extra].map(
|
|
112
|
+
({ instrumentName, kind }) =>
|
|
113
|
+
({
|
|
114
|
+
instrumentName,
|
|
115
|
+
aggregation: {
|
|
116
|
+
type: AggregationType.EXPLICIT_BUCKET_HISTOGRAM,
|
|
117
|
+
options: {
|
|
118
|
+
boundaries:
|
|
119
|
+
kind === 'duration'
|
|
120
|
+
? [...GEN_AI_DURATION_BUCKETS_SECONDS]
|
|
121
|
+
: kind === 'tokens'
|
|
122
|
+
? [...GEN_AI_TOKEN_USAGE_BUCKETS]
|
|
123
|
+
: [...GEN_AI_COST_USD_BUCKETS],
|
|
124
|
+
},
|
|
125
|
+
},
|
|
126
|
+
}) satisfies ViewOptions,
|
|
127
|
+
);
|
|
128
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
*/
|
|
30
30
|
|
|
31
31
|
// Core initialization
|
|
32
|
-
export { init, type AutotelConfig } from './init';
|
|
32
|
+
export { init, lockLogger, isLoggerLocked, type AutotelConfig } from './init';
|
|
33
33
|
|
|
34
34
|
// Baggage span processor
|
|
35
35
|
export {
|
|
@@ -126,6 +126,7 @@ export {
|
|
|
126
126
|
// Structured errors
|
|
127
127
|
export {
|
|
128
128
|
createStructuredError,
|
|
129
|
+
structuredErrorToJSON,
|
|
129
130
|
getStructuredErrorAttributes,
|
|
130
131
|
recordStructuredError,
|
|
131
132
|
type StructuredError,
|
|
@@ -186,6 +187,32 @@ export {
|
|
|
186
187
|
createObservableGauge,
|
|
187
188
|
} from './metric-helpers';
|
|
188
189
|
|
|
190
|
+
// LLM-tuned histogram buckets — pass genAiMetricViews() to your
|
|
191
|
+
// MeterProvider so gen_ai.* histograms have useful resolution.
|
|
192
|
+
export {
|
|
193
|
+
GEN_AI_DURATION_BUCKETS_SECONDS,
|
|
194
|
+
GEN_AI_TOKEN_USAGE_BUCKETS,
|
|
195
|
+
GEN_AI_COST_USD_BUCKETS,
|
|
196
|
+
genAiMetricViews,
|
|
197
|
+
llmHistogramAdvice,
|
|
198
|
+
} from './gen-ai-metrics';
|
|
199
|
+
|
|
200
|
+
// OTel GenAI span event helpers — record prompt-sent / response-received
|
|
201
|
+
// / retry / tool-call / stream-first-token as timestamped events aligned
|
|
202
|
+
// with the published GenAI semantic conventions.
|
|
203
|
+
export {
|
|
204
|
+
recordPromptSent,
|
|
205
|
+
recordResponseReceived,
|
|
206
|
+
recordRetry,
|
|
207
|
+
recordToolCall,
|
|
208
|
+
recordStreamFirstToken,
|
|
209
|
+
type PromptSentEvent,
|
|
210
|
+
type ResponseReceivedEvent,
|
|
211
|
+
type RetryEvent,
|
|
212
|
+
type ToolCallEvent,
|
|
213
|
+
type StreamFirstTokenEvent,
|
|
214
|
+
} from './gen-ai-events';
|
|
215
|
+
|
|
189
216
|
// Tracer helpers for custom spans
|
|
190
217
|
export {
|
|
191
218
|
getTracer,
|
package/src/init.ts
CHANGED
|
@@ -60,6 +60,7 @@ import {
|
|
|
60
60
|
} from './span-name-normalizer';
|
|
61
61
|
import {
|
|
62
62
|
AttributeRedactingProcessor,
|
|
63
|
+
normalizeAttributeRedactorConfig,
|
|
63
64
|
type AttributeRedactorConfig,
|
|
64
65
|
type AttributeRedactorPreset,
|
|
65
66
|
} from './attribute-redacting-processor';
|
|
@@ -1142,10 +1143,28 @@ export interface AutotelConfig {
|
|
|
1142
1143
|
*/
|
|
1143
1144
|
pretty?: boolean;
|
|
1144
1145
|
};
|
|
1146
|
+
|
|
1147
|
+
/**
|
|
1148
|
+
* Suppress console output while keeping OTel exporters running.
|
|
1149
|
+
* Useful for platforms like GCP Cloud Run / AWS Lambda where stdout
|
|
1150
|
+
* is managed externally by the platform's log collector.
|
|
1151
|
+
*
|
|
1152
|
+
* @default false
|
|
1153
|
+
*/
|
|
1154
|
+
silent?: boolean;
|
|
1155
|
+
|
|
1156
|
+
/**
|
|
1157
|
+
* Minimum log level for internal autotel diagnostic messages.
|
|
1158
|
+
* Messages below this level are dropped before processing.
|
|
1159
|
+
*
|
|
1160
|
+
* @default 'info'
|
|
1161
|
+
*/
|
|
1162
|
+
minLevel?: 'debug' | 'info' | 'warn' | 'error';
|
|
1145
1163
|
}
|
|
1146
1164
|
|
|
1147
1165
|
// Internal state
|
|
1148
1166
|
let initialized = false;
|
|
1167
|
+
let locked = false;
|
|
1149
1168
|
let config: AutotelConfig | null = null;
|
|
1150
1169
|
let sdk: NodeSDK | null = null;
|
|
1151
1170
|
let warnedOnce = false;
|
|
@@ -1156,6 +1175,85 @@ let _stringRedactor: StringRedactor | null = null;
|
|
|
1156
1175
|
let _optionalRequire: typeof safeRequire = safeRequire;
|
|
1157
1176
|
let _devtoolsClose: (() => Promise<void> | void) | null = null;
|
|
1158
1177
|
|
|
1178
|
+
const LOG_LEVELS = { debug: 0, info: 1, warn: 2, error: 3 } as const;
|
|
1179
|
+
type LogLevelKey = keyof typeof LOG_LEVELS;
|
|
1180
|
+
|
|
1181
|
+
/**
|
|
1182
|
+
* Lock the logger to prevent further `init()` calls.
|
|
1183
|
+
* Use this when framework plugins set up instrumentation and you want
|
|
1184
|
+
* to prevent accidental re-initialization from user code.
|
|
1185
|
+
*/
|
|
1186
|
+
export function lockLogger(): void {
|
|
1187
|
+
locked = true;
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
/**
|
|
1191
|
+
* Check if the logger has been locked.
|
|
1192
|
+
*/
|
|
1193
|
+
export function isLoggerLocked(): boolean {
|
|
1194
|
+
return locked;
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
function createSilentLogger(): Logger {
|
|
1198
|
+
return {
|
|
1199
|
+
info: () => {},
|
|
1200
|
+
warn: () => {},
|
|
1201
|
+
error: () => {},
|
|
1202
|
+
debug: () => {},
|
|
1203
|
+
};
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
function wrapLogger(
|
|
1207
|
+
base: Logger,
|
|
1208
|
+
silent: boolean,
|
|
1209
|
+
minLevel: LogLevelKey,
|
|
1210
|
+
): Logger {
|
|
1211
|
+
if (silent) return createSilentLogger();
|
|
1212
|
+
const threshold = LOG_LEVELS[minLevel];
|
|
1213
|
+
const wrap = (fn: Logger['info'], level: LogLevelKey): Logger['info'] => {
|
|
1214
|
+
if (LOG_LEVELS[level] < threshold) {
|
|
1215
|
+
return (() => {}) as Logger['info'];
|
|
1216
|
+
}
|
|
1217
|
+
return ((...args: Parameters<Logger['info']>) =>
|
|
1218
|
+
fn(...args)) as Logger['info'];
|
|
1219
|
+
};
|
|
1220
|
+
return {
|
|
1221
|
+
debug: wrap(base.debug, 'debug'),
|
|
1222
|
+
info: wrap(base.info, 'info'),
|
|
1223
|
+
warn: wrap(base.warn, 'warn'),
|
|
1224
|
+
error: wrap(base.error, 'error'),
|
|
1225
|
+
};
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
function detectEnvironmentAttributes(): Record<string, string> {
|
|
1229
|
+
const attrs: Record<string, string> = {};
|
|
1230
|
+
|
|
1231
|
+
const commitSha =
|
|
1232
|
+
process.env.COMMIT_SHA ||
|
|
1233
|
+
process.env.GITHUB_SHA ||
|
|
1234
|
+
process.env.VERCEL_GIT_COMMIT_SHA ||
|
|
1235
|
+
process.env.CF_PAGES_COMMIT_SHA ||
|
|
1236
|
+
process.env.AWS_CODEPIPELINE_EXECUTION_ID;
|
|
1237
|
+
if (commitSha) attrs['service.commit.sha'] = commitSha;
|
|
1238
|
+
|
|
1239
|
+
const region =
|
|
1240
|
+
process.env.VERCEL_REGION ||
|
|
1241
|
+
process.env.AWS_REGION ||
|
|
1242
|
+
process.env.AWS_DEFAULT_REGION ||
|
|
1243
|
+
process.env.FLY_REGION ||
|
|
1244
|
+
process.env.CF_REGION ||
|
|
1245
|
+
process.env.GOOGLE_CLOUD_REGION;
|
|
1246
|
+
if (region) attrs['service.region'] = region;
|
|
1247
|
+
|
|
1248
|
+
const version =
|
|
1249
|
+
process.env.APP_VERSION ||
|
|
1250
|
+
process.env.HEROKU_RELEASE_VERSION ||
|
|
1251
|
+
process.env.VERCEL_GIT_COMMIT_REF;
|
|
1252
|
+
if (version) attrs['service.deploy.version'] = version;
|
|
1253
|
+
|
|
1254
|
+
return attrs;
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1159
1257
|
/**
|
|
1160
1258
|
* Resolve metrics flag with env var override support
|
|
1161
1259
|
*/
|
|
@@ -1295,6 +1393,10 @@ function normalizeOtlpHeaders(
|
|
|
1295
1393
|
*/
|
|
1296
1394
|
|
|
1297
1395
|
export function init(cfg: AutotelConfig): void {
|
|
1396
|
+
if (locked) {
|
|
1397
|
+
return;
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1298
1400
|
// Resolve configs in priority order: explicit > yaml > env > defaults
|
|
1299
1401
|
const envConfig = resolveConfigFromEnv();
|
|
1300
1402
|
const yamlConfig = loadYamlConfig() ?? {};
|
|
@@ -1308,19 +1410,32 @@ export function init(cfg: AutotelConfig): void {
|
|
|
1308
1410
|
resourceAttributes: {
|
|
1309
1411
|
...envConfig.resourceAttributes,
|
|
1310
1412
|
...yamlConfig.resourceAttributes,
|
|
1413
|
+
...detectEnvironmentAttributes(),
|
|
1311
1414
|
...cfg.resourceAttributes,
|
|
1312
1415
|
},
|
|
1313
1416
|
// Handle headers merge (can be string or object)
|
|
1314
1417
|
headers: cfg.headers ?? yamlConfig.headers ?? envConfig.headers,
|
|
1315
1418
|
} as AutotelConfig;
|
|
1316
1419
|
|
|
1420
|
+
if (mergedConfig.attributeRedactor !== undefined) {
|
|
1421
|
+
const normalizedRedactor = normalizeAttributeRedactorConfig(
|
|
1422
|
+
mergedConfig.attributeRedactor,
|
|
1423
|
+
);
|
|
1424
|
+
if (!normalizedRedactor) {
|
|
1425
|
+
throw new Error('Invalid attributeRedactor config');
|
|
1426
|
+
}
|
|
1427
|
+
mergedConfig.attributeRedactor = normalizedRedactor;
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1317
1430
|
const devtoolsConfig = resolveDevtoolsConfig(mergedConfig.devtools);
|
|
1318
1431
|
if (devtoolsConfig.enabled && mergedConfig.logs === undefined) {
|
|
1319
1432
|
mergedConfig.logs = true;
|
|
1320
1433
|
}
|
|
1321
1434
|
|
|
1322
|
-
|
|
1323
|
-
|
|
1435
|
+
const silent = mergedConfig.silent ?? false;
|
|
1436
|
+
const minLevel = mergedConfig.minLevel ?? 'info';
|
|
1437
|
+
const baseLogger = mergedConfig.logger || silentLogger;
|
|
1438
|
+
logger = wrapLogger(baseLogger, silent, minLevel);
|
|
1324
1439
|
|
|
1325
1440
|
// Warn if re-initializing (same behavior in all environments)
|
|
1326
1441
|
if (initialized) {
|