@robota-sdk/agent-telemetry-otel 0.0.0-stage → 3.0.0-beta.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/LICENSE +661 -0
- package/README.md +130 -2
- package/dist/node/index.cjs +1 -0
- package/dist/node/index.d.cts +142 -0
- package/dist/node/index.d.cts.map +1 -0
- package/dist/node/index.d.ts +142 -0
- package/dist/node/index.d.ts.map +1 -0
- package/dist/node/index.js +2 -0
- package/dist/node/index.js.map +1 -0
- package/package.json +79 -3
- package/src/__tests__/metrics.test.ts +247 -0
- package/src/__tests__/projection.test.ts +262 -0
- package/src/__tests__/run-tracing.test.ts +230 -0
- package/src/index.ts +31 -0
- package/src/metrics.ts +140 -0
- package/src/projection.ts +216 -0
- package/src/run-tracing.ts +255 -0
- package/src/span-ids.ts +47 -0
- package/src/timing.ts +47 -0
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import { trace } from '@opentelemetry/api';
|
|
2
|
+
import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from '@opentelemetry/sdk-trace';
|
|
3
|
+
import { AbstractTool, ConversationAgent } from '@robota-sdk/agent-core';
|
|
4
|
+
import { createScriptedProvider } from '@robota-sdk/agent-core/testing';
|
|
5
|
+
import { afterEach, describe, expect, it } from 'vitest';
|
|
6
|
+
|
|
7
|
+
import { createOtelTracing, createStructuralIdGenerator } from '../index.js';
|
|
8
|
+
|
|
9
|
+
import type { TOtelProjectionMode } from '../index.js';
|
|
10
|
+
import type { SpanContext } from '@opentelemetry/api';
|
|
11
|
+
import type { ReadableSpan } from '@opentelemetry/sdk-trace';
|
|
12
|
+
import type {
|
|
13
|
+
IAgentConfig,
|
|
14
|
+
IToolExecutionContext,
|
|
15
|
+
IToolResult,
|
|
16
|
+
IToolSchema,
|
|
17
|
+
TToolParameters,
|
|
18
|
+
} from '@robota-sdk/agent-core';
|
|
19
|
+
|
|
20
|
+
const ORIGIN = 'https://model.example.com';
|
|
21
|
+
|
|
22
|
+
class LookupTool extends AbstractTool {
|
|
23
|
+
readonly contexts: IToolExecutionContext[] = [];
|
|
24
|
+
constructor(private readonly fail = false) {
|
|
25
|
+
super();
|
|
26
|
+
}
|
|
27
|
+
override get schema(): IToolSchema {
|
|
28
|
+
return { name: 'lookup', description: 'lookup', parameters: { type: 'object' as const, properties: {} } };
|
|
29
|
+
}
|
|
30
|
+
override async execute(parameters: TToolParameters, context: IToolExecutionContext): Promise<IToolResult> {
|
|
31
|
+
this.contexts.push(context);
|
|
32
|
+
return super.execute(parameters, context);
|
|
33
|
+
}
|
|
34
|
+
protected override async executeImpl(): Promise<IToolResult> {
|
|
35
|
+
if (this.fail) throw new Error('lookup failed');
|
|
36
|
+
return { success: true, data: { ok: true } };
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const cleanups: Array<() => Promise<void>> = [];
|
|
41
|
+
afterEach(async () => {
|
|
42
|
+
for (const cleanup of cleanups.splice(0)) await cleanup();
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
async function tracedRun(mode: TOtelProjectionMode, options: { fail?: boolean; ids?: boolean; observe?: boolean } = {}) {
|
|
46
|
+
const exporter = new InMemorySpanExporter();
|
|
47
|
+
const idGenerator = createStructuralIdGenerator();
|
|
48
|
+
const provider = new TracerProvider({
|
|
49
|
+
...(options.ids === false ? {} : { idGenerator }),
|
|
50
|
+
spanProcessors: [new SimpleSpanProcessor({ exporter })],
|
|
51
|
+
});
|
|
52
|
+
const tracer = provider.getTracer('test');
|
|
53
|
+
const tracing = createOtelTracing({ tracer, mode, ...(options.ids === false ? {} : { idGenerator }) });
|
|
54
|
+
const tool = new LookupTool(options.fail);
|
|
55
|
+
const scripted = createScriptedProvider([
|
|
56
|
+
{ toolCalls: [{ name: 'lookup', args: {} }], usage: { inputTokens: 12, outputTokens: 3 } },
|
|
57
|
+
{ text: 'done' },
|
|
58
|
+
]);
|
|
59
|
+
// The scripted double neither carries trace context nor attests usage; this one does both, so the
|
|
60
|
+
// sent `traceparent` and the token counts are observable.
|
|
61
|
+
const chat = scripted.provider.chat.bind(scripted.provider);
|
|
62
|
+
const aiProvider = Object.assign(scripted.provider, {
|
|
63
|
+
canPropagateTraceContext: () => true,
|
|
64
|
+
chat: async (...args: Parameters<typeof chat>) => {
|
|
65
|
+
const message = await chat(...args);
|
|
66
|
+
return message.metadata?.['inputTokens'] === undefined
|
|
67
|
+
? message : { ...message, metadata: { ...message.metadata, usageProvenance: 'complete' } };
|
|
68
|
+
},
|
|
69
|
+
});
|
|
70
|
+
const agent = new ConversationAgent({
|
|
71
|
+
name: 'traced',
|
|
72
|
+
aiProviders: [aiProvider],
|
|
73
|
+
defaultModel: { provider: 'scripted-test-provider', model: 'test-model' },
|
|
74
|
+
tools: [tool],
|
|
75
|
+
plugins: [tracing.plugin],
|
|
76
|
+
logging: { level: 'silent', enabled: false },
|
|
77
|
+
} as IAgentConfig);
|
|
78
|
+
cleanups.push(async () => {
|
|
79
|
+
await agent.destroy();
|
|
80
|
+
await provider.shutdown();
|
|
81
|
+
});
|
|
82
|
+
const root = tracer.startSpan('invoke_agent traced');
|
|
83
|
+
const run = tracing.startRun(root.spanContext(), { allowedOrigins: [ORIGIN], subprocessClasses: ['shell'] });
|
|
84
|
+
await agent.run('go', {
|
|
85
|
+
...(run.traceContext ? { traceContext: run.traceContext } : {}),
|
|
86
|
+
...(options.observe === false ? {} : { onExecutionEvent: run.onExecutionEvent }),
|
|
87
|
+
});
|
|
88
|
+
run.end();
|
|
89
|
+
root.end();
|
|
90
|
+
const spans = exporter.getFinishedSpans();
|
|
91
|
+
const sent = scripted.chatOptions.map((chat) => chat?.outboundTraceContext?.traceparent);
|
|
92
|
+
return { spans, sent, tool, root: root.spanContext(), tracing, tracer, exporter, agent };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const byName = (spans: ReadableSpan[], prefix: string) => spans.filter((span) => span.name.startsWith(prefix));
|
|
96
|
+
const spanOf = (traceparent: string | undefined) => traceparent?.split('-')[2];
|
|
97
|
+
|
|
98
|
+
describe('createOtelTracing', () => {
|
|
99
|
+
it('records provider calls and tool bodies under the host span, with the IDs Robota propagated', async () => {
|
|
100
|
+
const { spans, sent, tool, root } = await tracedRun('gen_ai');
|
|
101
|
+
const chats = byName(spans, 'chat');
|
|
102
|
+
const tools = byName(spans, 'execute_tool');
|
|
103
|
+
expect(chats.map((span) => span.name)).toEqual(['chat test-model', 'chat test-model']);
|
|
104
|
+
expect(tools.map((span) => span.name)).toEqual(['execute_tool lookup']);
|
|
105
|
+
for (const span of [...chats, ...tools]) {
|
|
106
|
+
expect(span.spanContext().traceId).toBe(root.traceId);
|
|
107
|
+
expect(span.parentSpanContext?.spanId).toBe(root.spanId);
|
|
108
|
+
}
|
|
109
|
+
// The span each recipient was told is its parent is the span the trace contains.
|
|
110
|
+
expect(chats.map((span) => span.spanContext().spanId)).toEqual(sent.map(spanOf));
|
|
111
|
+
expect(tools[0]?.spanContext().spanId).toBe(spanOf(tool.contexts[0]?.shellTraceEnv?.TRACEPARENT));
|
|
112
|
+
expect(chats[0]?.attributes).toMatchObject({
|
|
113
|
+
'gen_ai.operation.name': 'chat',
|
|
114
|
+
'gen_ai.request.model': 'test-model',
|
|
115
|
+
'gen_ai.usage.input_tokens': 12,
|
|
116
|
+
'gen_ai.usage.output_tokens': 3,
|
|
117
|
+
'agent.provider.round': 1,
|
|
118
|
+
});
|
|
119
|
+
expect(tools[0]?.attributes).toMatchObject({
|
|
120
|
+
'gen_ai.operation.name': 'execute_tool', 'gen_ai.tool.name': 'lookup', 'agent.outcome': 'success',
|
|
121
|
+
});
|
|
122
|
+
expect(tools[0]?.attributes['gen_ai.tool.call.id']).toEqual(expect.any(String));
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
it('names spans as the projection does in agent mode', async () => {
|
|
126
|
+
const { spans } = await tracedRun('agent');
|
|
127
|
+
expect(spans.map((span) => span.name).sort()).toEqual([
|
|
128
|
+
'agent.provider_call', 'agent.provider_call', 'agent.tool_body', 'invoke_agent traced',
|
|
129
|
+
]);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it('records no tool body for a run it was not told about', async () => {
|
|
133
|
+
const { spans } = await tracedRun('gen_ai', { observe: false });
|
|
134
|
+
expect(spans.map((span) => span.name)).toEqual(['invoke_agent traced']);
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
it('marks a failed tool body as an error', async () => {
|
|
138
|
+
const { spans } = await tracedRun('gen_ai', { fail: true });
|
|
139
|
+
const [tool] = byName(spans, 'execute_tool');
|
|
140
|
+
expect(tool?.status.code).toBe(2);
|
|
141
|
+
expect(tool?.attributes).toMatchObject({ 'agent.outcome': 'failure', 'error.type': '_OTHER' });
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
it('keeps trace and parent without the ID generator, and says nothing about span IDs', async () => {
|
|
145
|
+
const { spans, sent, root } = await tracedRun('gen_ai', { ids: false });
|
|
146
|
+
const chats = byName(spans, 'chat');
|
|
147
|
+
expect(chats).toHaveLength(2);
|
|
148
|
+
for (const span of chats) {
|
|
149
|
+
expect(span.spanContext().traceId).toBe(root.traceId);
|
|
150
|
+
expect(span.parentSpanContext?.spanId).toBe(root.spanId);
|
|
151
|
+
}
|
|
152
|
+
expect(chats.map((span) => span.spanContext().spanId)).not.toEqual(sent.map(spanOf));
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
it('credits a nested agent\'s tool body to the nested run, not the traced parent run', async () => {
|
|
156
|
+
const exporter = new InMemorySpanExporter();
|
|
157
|
+
const provider = new TracerProvider({ spanProcessors: [new SimpleSpanProcessor({ exporter })] });
|
|
158
|
+
const tracer = provider.getTracer('test');
|
|
159
|
+
const tracing = createOtelTracing({ tracer, mode: 'gen_ai' });
|
|
160
|
+
const leaf = new LookupTool();
|
|
161
|
+
const config = (name: string, tools: AbstractTool[], script: ReturnType<typeof createScriptedProvider>) => ({
|
|
162
|
+
name, aiProviders: [script.provider], defaultModel: { provider: 'scripted-test-provider', model: 'm' },
|
|
163
|
+
tools, plugins: [tracing.plugin], logging: { level: 'silent', enabled: false },
|
|
164
|
+
});
|
|
165
|
+
class DelegateTool extends LookupTool {
|
|
166
|
+
override get schema(): IToolSchema {
|
|
167
|
+
return { name: 'delegate', description: 'delegate', parameters: { type: 'object' as const, properties: {} } };
|
|
168
|
+
}
|
|
169
|
+
override async execute(parameters: TToolParameters, context: IToolExecutionContext): Promise<IToolResult> {
|
|
170
|
+
// An agent built inside a tool keeps the caller's owner path, as IAgentConfig documents.
|
|
171
|
+
const child = new ConversationAgent({
|
|
172
|
+
...config('child', [leaf], createScriptedProvider([{ toolCalls: [{ name: 'lookup', args: {} }] }, { text: 'ok' }])),
|
|
173
|
+
executionContext: { ownerPath: context.ownerPath ?? [] },
|
|
174
|
+
} as IAgentConfig);
|
|
175
|
+
await child.run('child');
|
|
176
|
+
await child.destroy();
|
|
177
|
+
return super.execute(parameters, context);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
const parent = new ConversationAgent(config('parent', [new DelegateTool()],
|
|
181
|
+
createScriptedProvider([{ toolCalls: [{ name: 'delegate', args: {} }] }, { text: 'done' }])) as IAgentConfig);
|
|
182
|
+
const root = tracer.startSpan('root');
|
|
183
|
+
const run = tracing.startRun(root.spanContext());
|
|
184
|
+
await parent.run('go', { ...(run.traceContext ? { traceContext: run.traceContext } : {}), onExecutionEvent: run.onExecutionEvent });
|
|
185
|
+
run.end();
|
|
186
|
+
root.end();
|
|
187
|
+
await parent.destroy();
|
|
188
|
+
expect(leaf.contexts).toHaveLength(1);
|
|
189
|
+
expect(byName(exporter.getFinishedSpans(), 'execute_tool').map((span) => span.name)).toEqual(['execute_tool delegate']);
|
|
190
|
+
await provider.shutdown();
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
it('traces nothing and propagates nothing under a parent the host did not sample', () => {
|
|
194
|
+
const tracing = createOtelTracing({ tracer: trace.getTracer('noop'), mode: 'gen_ai' });
|
|
195
|
+
const run = tracing.startRun({ traceId: 'a'.repeat(32), spanId: 'b'.repeat(16), traceFlags: 0 });
|
|
196
|
+
expect(run.traceContext).toBeUndefined();
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
it('traces nothing and propagates nothing under an invalid parent', async () => {
|
|
200
|
+
const tracing = createOtelTracing({ tracer: trace.getTracer('noop'), mode: 'gen_ai' });
|
|
201
|
+
const invalid: SpanContext = { traceId: '0'.repeat(32), spanId: '0'.repeat(16), traceFlags: 0 };
|
|
202
|
+
const run = tracing.startRun(invalid);
|
|
203
|
+
expect(run.traceContext).toBeUndefined();
|
|
204
|
+
expect(() => run.onExecutionEvent('provider_call_completed', {})).not.toThrow();
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it('ignores a malformed provider-call event instead of failing the run', async () => {
|
|
208
|
+
const exporter = new InMemorySpanExporter();
|
|
209
|
+
const provider = new TracerProvider({ spanProcessors: [new SimpleSpanProcessor({ exporter })] });
|
|
210
|
+
const tracer = provider.getTracer('test');
|
|
211
|
+
const tracing = createOtelTracing({ tracer, mode: 'gen_ai' });
|
|
212
|
+
const root = tracer.startSpan('root');
|
|
213
|
+
const run = tracing.startRun(root.spanContext());
|
|
214
|
+
run.onExecutionEvent('provider_call_completed', { executionId: 'e1', callId: 'not hex', outcome: 'success' });
|
|
215
|
+
run.onExecutionEvent('provider_call_completed', { executionId: 'e1', callId: '00000000-0000-4000-8000-000000000000' });
|
|
216
|
+
expect(exporter.getFinishedSpans()).toEqual([]);
|
|
217
|
+
await provider.shutdown();
|
|
218
|
+
});
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
describe('createStructuralIdGenerator', () => {
|
|
222
|
+
it('hands the chosen ID to exactly one span and random IDs otherwise', () => {
|
|
223
|
+
const ids = createStructuralIdGenerator();
|
|
224
|
+
expect(ids.withSpanId('1234567890abcdef', () => ids.generateSpanId())).toBe('1234567890abcdef');
|
|
225
|
+
expect(ids.generateSpanId()).toMatch(/^(?!0{16}$)[0-9a-f]{16}$/u);
|
|
226
|
+
ids.withSpanId('1234567890abcdef', () => undefined);
|
|
227
|
+
expect(ids.generateSpanId()).not.toBe('1234567890abcdef');
|
|
228
|
+
expect(ids.generateTraceId()).toMatch(/^(?!0{32}$)[0-9a-f]{32}$/u);
|
|
229
|
+
});
|
|
230
|
+
});
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export {
|
|
2
|
+
GEN_AI_SCHEMA_URL,
|
|
3
|
+
projectLivePromptTrace,
|
|
4
|
+
type IOtelProjectionOptions,
|
|
5
|
+
type IOtelSpanProjection,
|
|
6
|
+
type IOtelTraceProjection,
|
|
7
|
+
type TOtelAttributes,
|
|
8
|
+
type TOtelProjectionMode,
|
|
9
|
+
type TOtelSpanKind,
|
|
10
|
+
} from './projection.js';
|
|
11
|
+
export {
|
|
12
|
+
GEN_AI_AGENT_DURATION_BUCKETS_S,
|
|
13
|
+
GEN_AI_CALL_COUNT_BUCKETS,
|
|
14
|
+
GEN_AI_DURATION_BUCKETS_S,
|
|
15
|
+
GEN_AI_TOKEN_BUCKETS,
|
|
16
|
+
projectLivePromptMetricSamples,
|
|
17
|
+
type IOtelMetricSample,
|
|
18
|
+
type IOtelMetricSampleOptions,
|
|
19
|
+
} from './metrics.js';
|
|
20
|
+
export {
|
|
21
|
+
createOtelTracing,
|
|
22
|
+
type IOtelRunPropagation,
|
|
23
|
+
type IOtelRunTracing,
|
|
24
|
+
type IOtelTracing,
|
|
25
|
+
type IOtelTracingOptions,
|
|
26
|
+
} from './run-tracing.js';
|
|
27
|
+
export {
|
|
28
|
+
createStructuralIdGenerator,
|
|
29
|
+
type IOtelIdGenerator,
|
|
30
|
+
type IStructuralIdGenerator,
|
|
31
|
+
} from './span-ids.js';
|
package/src/metrics.ts
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import type { ILivePromptTraceBatch, IProviderCallTraceEntry } from '@robota-sdk/agent-interface-analytics';
|
|
2
|
+
|
|
3
|
+
import { genAiProviderName, label } from './projection.js';
|
|
4
|
+
import type { TOtelAttributes, TOtelProjectionMode } from './projection.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Bucket boundaries the GenAI conventions recommend, as pinned at semantic-conventions-genai
|
|
8
|
+
* f3c6d0c9bb66ebd474668a3a0be25c61226f02c2. A histogram needs one more count than it has boundaries,
|
|
9
|
+
* for values above the last one.
|
|
10
|
+
*/
|
|
11
|
+
export const GEN_AI_DURATION_BUCKETS_S: readonly number[] =
|
|
12
|
+
[0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92];
|
|
13
|
+
export const GEN_AI_AGENT_DURATION_BUCKETS_S: readonly number[] =
|
|
14
|
+
[0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 102.4, 204.8, 409.6];
|
|
15
|
+
export const GEN_AI_TOKEN_BUCKETS: readonly number[] =
|
|
16
|
+
[1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864];
|
|
17
|
+
export const GEN_AI_CALL_COUNT_BUCKETS: readonly number[] = [1, 2, 4, 8, 16, 32, 64, 128];
|
|
18
|
+
|
|
19
|
+
/** One measurement to record, independent of any OpenTelemetry SDK: a counter increment or a histogram value. */
|
|
20
|
+
export interface IOtelMetricSample {
|
|
21
|
+
readonly name: string;
|
|
22
|
+
readonly instrument: 'counter' | 'histogram';
|
|
23
|
+
readonly unit: string;
|
|
24
|
+
readonly value: number;
|
|
25
|
+
readonly attributes: TOtelAttributes;
|
|
26
|
+
/** Explicit bucket upper bounds of a histogram; absent for a counter. */
|
|
27
|
+
readonly boundaries?: readonly number[];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface IOtelMetricSampleOptions {
|
|
31
|
+
/** The GenAI attribute set in use, as for spans: `gen_ai/dup` also keeps `agent.model.id`, `gen_ai` does not. */
|
|
32
|
+
readonly mode: Exclude<TOtelProjectionMode, 'agent'>;
|
|
33
|
+
/** Host-chosen agent name carried on the agent, tool and root metrics. */
|
|
34
|
+
readonly agentName?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Same fallback value the span projection uses: the batch records that a call failed, not why. */
|
|
38
|
+
const ERROR_TYPE = { 'error.type': '_OTHER' } as const;
|
|
39
|
+
|
|
40
|
+
const seconds = (startedAt: string, endedAt: string): number | undefined => {
|
|
41
|
+
const value = (Date.parse(endedAt) - Date.parse(startedAt)) / 1000;
|
|
42
|
+
return Number.isFinite(value) && value >= 0 ? value : undefined;
|
|
43
|
+
};
|
|
44
|
+
const tokenCount = (value: unknown): number | undefined =>
|
|
45
|
+
typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined;
|
|
46
|
+
|
|
47
|
+
function providerCallSamples(trace: IProviderCallTraceEntry, mode: IOtelMetricSampleOptions['mode']): IOtelMetricSample[] {
|
|
48
|
+
const providerName = genAiProviderName(trace.providerId);
|
|
49
|
+
const providerId = label(trace.providerId);
|
|
50
|
+
const model = label(trace.modelId);
|
|
51
|
+
// Same rule as the provider span: a fact the conventions do not define keeps its `agent.*` name, so an
|
|
52
|
+
// operator's provider split survives (a provider outside the known map has no `gen_ai.provider.name`);
|
|
53
|
+
// the model has a convention name, so `agent.model.id` is repeated only while both sets are emitted.
|
|
54
|
+
const base: TOtelAttributes = {
|
|
55
|
+
'gen_ai.operation.name': 'chat',
|
|
56
|
+
...(providerName ? { 'gen_ai.provider.name': providerName } : {}),
|
|
57
|
+
...(model ? { 'gen_ai.request.model': model } : {}),
|
|
58
|
+
...(providerId ? { 'agent.provider.id': providerId } : {}),
|
|
59
|
+
...(mode === 'gen_ai/dup' && model ? { 'agent.model.id': model } : {}),
|
|
60
|
+
};
|
|
61
|
+
const attributes: TOtelAttributes = trace.outcome === 'failure' ? { ...base, ...ERROR_TYPE } : base;
|
|
62
|
+
// Without a complete count the conventions ask for 0, not for a missing sample; providers do not
|
|
63
|
+
// report usage per modality, so the counters' modality is `unknown`.
|
|
64
|
+
const promptTokens = tokenCount(trace.promptTokens);
|
|
65
|
+
const completionTokens = tokenCount(trace.completionTokens);
|
|
66
|
+
const known = trace.usageProvenance === 'complete' && promptTokens !== undefined && completionTokens !== undefined;
|
|
67
|
+
const input = known ? promptTokens : 0;
|
|
68
|
+
const output = known ? completionTokens : 0;
|
|
69
|
+
const counterAttributes: TOtelAttributes = { ...base, 'gen_ai.token.modality': 'unknown' };
|
|
70
|
+
const duration = seconds(trace.startedAt, trace.endedAt);
|
|
71
|
+
const tokens = (name: string, value: number): IOtelMetricSample => ({
|
|
72
|
+
name, instrument: 'histogram', unit: '{token}', value, attributes, boundaries: GEN_AI_TOKEN_BUCKETS,
|
|
73
|
+
});
|
|
74
|
+
return [
|
|
75
|
+
...(duration === undefined ? [] : [{
|
|
76
|
+
name: 'gen_ai.client.inference.duration', instrument: 'histogram' as const, unit: 's', value: duration,
|
|
77
|
+
attributes, boundaries: GEN_AI_DURATION_BUCKETS_S,
|
|
78
|
+
}]),
|
|
79
|
+
{ name: 'gen_ai.client.inference.usage.input_tokens', instrument: 'counter', unit: '{token}', value: input, attributes: counterAttributes },
|
|
80
|
+
{ name: 'gen_ai.client.inference.usage.output_tokens', instrument: 'counter', unit: '{token}', value: output, attributes: counterAttributes },
|
|
81
|
+
tokens('gen_ai.client.inference.operation.input_tokens', input),
|
|
82
|
+
tokens('gen_ai.client.inference.operation.output_tokens', output),
|
|
83
|
+
];
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Map one live prompt trace batch onto the GenAI metrics a host records: one inference measurement set
|
|
88
|
+
* per invoked provider call, one duration per tool body, and the prompt's agent-level duration and call
|
|
89
|
+
* counts. Provider-derived samples are left out when the batch omitted provider children, and
|
|
90
|
+
* tool-derived ones when it omitted tool children, because a bounded child list cannot prove a complete
|
|
91
|
+
* total. Cache hits and refused calls were not inferences, and permission decisions have no duration, so
|
|
92
|
+
* they produce nothing. Like the span projection, it is pure and content-free.
|
|
93
|
+
*/
|
|
94
|
+
export function projectLivePromptMetricSamples(
|
|
95
|
+
batch: ILivePromptTraceBatch, options: IOtelMetricSampleOptions,
|
|
96
|
+
): IOtelMetricSample[] {
|
|
97
|
+
const agent: TOtelAttributes = options.agentName ? { 'gen_ai.agent.name': options.agentName } : {};
|
|
98
|
+
const samples: IOtelMetricSample[] = [];
|
|
99
|
+
const providerComplete = batch.omittedChildren.provider === 0;
|
|
100
|
+
const toolComplete = batch.omittedChildren.tool === 0;
|
|
101
|
+
let invokedCalls = 0;
|
|
102
|
+
let toolBodies = 0;
|
|
103
|
+
|
|
104
|
+
for (const child of batch.children) {
|
|
105
|
+
if (child.kind !== 'provider' || child.trace.disposition !== 'invoked') continue;
|
|
106
|
+
invokedCalls += 1;
|
|
107
|
+
if (providerComplete) samples.push(...providerCallSamples(child.trace, options.mode));
|
|
108
|
+
}
|
|
109
|
+
for (const child of batch.children) {
|
|
110
|
+
if (child.kind !== 'tool') continue;
|
|
111
|
+
toolBodies += 1;
|
|
112
|
+
const duration = toolComplete ? seconds(child.trace.startedAt, child.trace.endedAt) : undefined;
|
|
113
|
+
if (duration === undefined) continue;
|
|
114
|
+
const toolName = label(child.trace.toolName);
|
|
115
|
+
samples.push({
|
|
116
|
+
name: 'gen_ai.execute_tool.duration', instrument: 'histogram', unit: 's', value: duration,
|
|
117
|
+
boundaries: GEN_AI_DURATION_BUCKETS_S,
|
|
118
|
+
attributes: {
|
|
119
|
+
...(toolName ? { 'gen_ai.tool.name': toolName } : {}), ...agent,
|
|
120
|
+
...(child.trace.outcome === 'failure' ? ERROR_TYPE : {}),
|
|
121
|
+
},
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const rootDuration = seconds(batch.root.startedAt, batch.root.endedAt);
|
|
126
|
+
if (rootDuration !== undefined) samples.push({
|
|
127
|
+
name: 'gen_ai.invoke_agent.duration', instrument: 'histogram', unit: 's', value: rootDuration,
|
|
128
|
+
boundaries: GEN_AI_AGENT_DURATION_BUCKETS_S,
|
|
129
|
+
attributes: { ...agent, ...(batch.root.outcome === 'failure' ? ERROR_TYPE : {}) },
|
|
130
|
+
});
|
|
131
|
+
if (providerComplete) samples.push({
|
|
132
|
+
name: 'gen_ai.invoke_agent.inference_calls', instrument: 'histogram', unit: '{inference_call}',
|
|
133
|
+
value: invokedCalls, attributes: agent, boundaries: GEN_AI_CALL_COUNT_BUCKETS,
|
|
134
|
+
});
|
|
135
|
+
if (toolComplete) samples.push({
|
|
136
|
+
name: 'gen_ai.invoke_agent.tool_calls', instrument: 'histogram', unit: '{tool_call}',
|
|
137
|
+
value: toolBodies, attributes: agent, boundaries: GEN_AI_CALL_COUNT_BUCKETS,
|
|
138
|
+
});
|
|
139
|
+
return samples;
|
|
140
|
+
}
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
ILivePromptTraceBatch,
|
|
3
|
+
IProviderCallTraceEntry,
|
|
4
|
+
IToolBodyTraceEntry,
|
|
5
|
+
} from '@robota-sdk/agent-interface-analytics';
|
|
6
|
+
|
|
7
|
+
import { timingAttributes } from './timing.js';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Which names a projected span carries. `agent` keeps Robota's own `agent.*` names; `gen_ai` uses the
|
|
11
|
+
* OpenTelemetry GenAI semantic conventions; `gen_ai/dup` uses GenAI span names and emits both attribute
|
|
12
|
+
* sets, so a backend can move from one set of names to the other without a gap.
|
|
13
|
+
*/
|
|
14
|
+
export type TOtelProjectionMode = 'agent' | 'gen_ai/dup' | 'gen_ai';
|
|
15
|
+
|
|
16
|
+
export interface IOtelProjectionOptions {
|
|
17
|
+
readonly mode: TOtelProjectionMode;
|
|
18
|
+
/** Host-chosen agent name for the root `invoke_agent` span; GenAI modes only. */
|
|
19
|
+
readonly agentName?: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export type TOtelSpanKind = 'internal' | 'client';
|
|
23
|
+
export type TOtelAttributes = Readonly<Record<string, string | number>>;
|
|
24
|
+
|
|
25
|
+
/** One span to record, independent of any OpenTelemetry SDK. */
|
|
26
|
+
export interface IOtelSpanProjection {
|
|
27
|
+
readonly name: string;
|
|
28
|
+
readonly kind: TOtelSpanKind;
|
|
29
|
+
readonly traceId: string;
|
|
30
|
+
readonly spanId: string;
|
|
31
|
+
/** Absent on the root; every child's parent is the root. */
|
|
32
|
+
readonly parentSpanId?: string;
|
|
33
|
+
readonly startedAt: string;
|
|
34
|
+
readonly endedAt: string;
|
|
35
|
+
/** `error` for a failed outcome; an interrupted one is not an error. */
|
|
36
|
+
readonly status: 'unset' | 'error';
|
|
37
|
+
readonly attributes: TOtelAttributes;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface IOtelTraceProjection {
|
|
41
|
+
/** The schema the attribute names follow; absent in `agent` mode, whose names are Robota's own. */
|
|
42
|
+
readonly schemaUrl?: string;
|
|
43
|
+
readonly root: IOtelSpanProjection;
|
|
44
|
+
readonly children: readonly IOtelSpanProjection[];
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** The GenAI semantic-convention version the `gen_ai` names follow. */
|
|
48
|
+
export const GEN_AI_SCHEMA_URL = 'https://opentelemetry.io/schemas/gen-ai-dev/1.42.0-dev';
|
|
49
|
+
|
|
50
|
+
/** The same opaque-ID shape the framework's live-prompt-trace boundary allows through. */
|
|
51
|
+
const OPAQUE_ID = /^[A-Za-z0-9_-]{1,128}$/u;
|
|
52
|
+
/** The same label shape the framework allows for provider and model labels. */
|
|
53
|
+
const LABEL = /^[A-Za-z0-9._:/-]{1,128}$/u;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Robota provider IDs whose GenAI provider name is certain. A Gemini client can point at either the
|
|
57
|
+
* Gemini API or Vertex AI, so it gets the convention's name for any Google GenAI endpoint. An ID not
|
|
58
|
+
* listed here gets no `gen_ai.provider.name` rather than a guessed one; `agent.provider.id` still names it.
|
|
59
|
+
*/
|
|
60
|
+
const GEN_AI_PROVIDER_NAMES: Readonly<Record<string, string>> = {
|
|
61
|
+
anthropic: 'anthropic',
|
|
62
|
+
openai: 'openai',
|
|
63
|
+
deepseek: 'deepseek',
|
|
64
|
+
gemini: 'gcp.gen_ai',
|
|
65
|
+
google: 'gcp.gen_ai',
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
const opaqueId = (value: unknown): string | undefined =>
|
|
69
|
+
typeof value === 'string' && OPAQUE_ID.test(value) ? value : undefined;
|
|
70
|
+
/** A model or tool name only in the label shape the framework allows; anything else is dropped. */
|
|
71
|
+
export const label = (value: unknown): string | undefined =>
|
|
72
|
+
typeof value === 'string' && LABEL.test(value) ? value : undefined;
|
|
73
|
+
/** Own keys only: an inherited property such as `constructor` is not a provider. */
|
|
74
|
+
export const genAiProviderName = (providerId: string | undefined): string | undefined =>
|
|
75
|
+
providerId !== undefined && Object.hasOwn(GEN_AI_PROVIDER_NAMES, providerId) ? GEN_AI_PROVIDER_NAMES[providerId] : undefined;
|
|
76
|
+
const status = (outcome: string): 'unset' | 'error' => outcome === 'failure' ? 'error' : 'unset';
|
|
77
|
+
const spanName = (operation: string, subject: string | undefined): string =>
|
|
78
|
+
subject ? `${operation} ${subject}` : operation;
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* GenAI conventions require `error.type` on a failed operation. The batch records that a span failed
|
|
82
|
+
* but not why, so the projection uses the conventions' fallback value instead of naming a cause.
|
|
83
|
+
*/
|
|
84
|
+
function withErrorType(attributes: TOtelAttributes, outcome: string, mode: TOtelProjectionMode): TOtelAttributes {
|
|
85
|
+
return mode !== 'agent' && outcome === 'failure' ? { ...attributes, 'error.type': '_OTHER' } : attributes;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** A provider-call or tool-body child of a trace: the children that get a span. */
|
|
89
|
+
export type TSpannedChild = Exclude<ILivePromptTraceBatch['children'][number], { readonly kind: 'permission' }>;
|
|
90
|
+
|
|
91
|
+
/** A permission decision is content-free but has no duration of its own — it never gets a span. */
|
|
92
|
+
function hasSpan(child: ILivePromptTraceBatch['children'][number]): child is TSpannedChild {
|
|
93
|
+
return child.kind !== 'permission';
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function rootAttributes(batch: ILivePromptTraceBatch, options: IOtelProjectionOptions): TOtelAttributes {
|
|
97
|
+
const shared = {
|
|
98
|
+
'agent.turn.id': batch.turnId,
|
|
99
|
+
'agent.outcome': batch.root.outcome,
|
|
100
|
+
'agent.omitted.provider_count': batch.omittedChildren.provider,
|
|
101
|
+
'agent.omitted.tool_count': batch.omittedChildren.tool,
|
|
102
|
+
'agent.omitted.permission_count': batch.omittedChildren.permission,
|
|
103
|
+
...timingAttributes(batch.timingTotals),
|
|
104
|
+
};
|
|
105
|
+
const agent = { 'agent.session.id': batch.sessionId };
|
|
106
|
+
const genAi = {
|
|
107
|
+
'gen_ai.operation.name': 'invoke_agent',
|
|
108
|
+
...(options.agentName ? { 'gen_ai.agent.name': options.agentName } : {}),
|
|
109
|
+
'gen_ai.conversation.id': batch.sessionId,
|
|
110
|
+
};
|
|
111
|
+
if (options.mode === 'agent') return { ...agent, ...shared };
|
|
112
|
+
return options.mode === 'gen_ai' ? { ...genAi, ...shared } : { ...genAi, ...agent, ...shared };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function providerAttributes(trace: IProviderCallTraceEntry, mode: TOtelProjectionMode): TOtelAttributes {
|
|
116
|
+
const complete = trace.usageProvenance === 'complete';
|
|
117
|
+
const requestId = opaqueId(trace.providerRequestId);
|
|
118
|
+
const agentOnly = mode !== 'gen_ai';
|
|
119
|
+
// The CLI's attribute order, with the facts GenAI names differently present only outside `gen_ai`.
|
|
120
|
+
const robota = {
|
|
121
|
+
'agent.outcome': trace.outcome,
|
|
122
|
+
'agent.provider.round': trace.round,
|
|
123
|
+
...(trace.disposition ? { 'agent.provider.disposition': trace.disposition } : {}),
|
|
124
|
+
...(trace.providerId ? { 'agent.provider.id': trace.providerId } : {}),
|
|
125
|
+
...(agentOnly && trace.modelId ? { 'agent.model.id': trace.modelId } : {}),
|
|
126
|
+
...(trace.usageProvenance ? { 'agent.usage.provenance': trace.usageProvenance } : {}),
|
|
127
|
+
...(complete ? {
|
|
128
|
+
...(agentOnly ? {
|
|
129
|
+
'agent.usage.input_tokens': trace.promptTokens ?? 0,
|
|
130
|
+
'agent.usage.output_tokens': trace.completionTokens ?? 0,
|
|
131
|
+
} : {}),
|
|
132
|
+
// GenAI conventions define input and output tokens but no total.
|
|
133
|
+
'agent.usage.total_tokens': trace.totalTokens ?? 0,
|
|
134
|
+
} : {}),
|
|
135
|
+
...(requestId ? { 'agent.provider.request_id': requestId } : {}),
|
|
136
|
+
};
|
|
137
|
+
const providerName = genAiProviderName(trace.providerId);
|
|
138
|
+
const genAi = {
|
|
139
|
+
'gen_ai.operation.name': 'chat',
|
|
140
|
+
...(providerName ? { 'gen_ai.provider.name': providerName } : {}),
|
|
141
|
+
...(trace.modelId ? { 'gen_ai.request.model': trace.modelId } : {}),
|
|
142
|
+
...(complete ? {
|
|
143
|
+
'gen_ai.usage.input_tokens': trace.promptTokens ?? 0,
|
|
144
|
+
'gen_ai.usage.output_tokens': trace.completionTokens ?? 0,
|
|
145
|
+
} : {}),
|
|
146
|
+
};
|
|
147
|
+
return mode === 'agent' ? robota : { ...genAi, ...robota };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function toolAttributes(trace: IToolBodyTraceEntry, mode: TOtelProjectionMode): TOtelAttributes {
|
|
151
|
+
const callId = opaqueId(trace.toolCallId);
|
|
152
|
+
const toolName = label(trace.toolName);
|
|
153
|
+
const shared = { 'agent.outcome': trace.outcome };
|
|
154
|
+
const agent: TOtelAttributes = callId ? { 'agent.tool.call_id': callId } : {};
|
|
155
|
+
const genAi = {
|
|
156
|
+
'gen_ai.operation.name': 'execute_tool',
|
|
157
|
+
...(toolName ? { 'gen_ai.tool.name': toolName } : {}),
|
|
158
|
+
...(callId ? { 'gen_ai.tool.call.id': callId } : {}),
|
|
159
|
+
};
|
|
160
|
+
if (mode === 'agent') return { ...shared, ...agent };
|
|
161
|
+
return mode === 'gen_ai' ? { ...genAi, ...shared } : { ...genAi, ...shared, ...agent };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** One child's span under `parentSpanId` in trace `traceId`; shared by the batch projection and the run observer. */
|
|
165
|
+
export function projectTraceChild(
|
|
166
|
+
child: TSpannedChild, traceId: string, parentSpanId: string, mode: TOtelProjectionMode,
|
|
167
|
+
): IOtelSpanProjection {
|
|
168
|
+
const { trace } = child;
|
|
169
|
+
const common = {
|
|
170
|
+
traceId,
|
|
171
|
+
spanId: trace.spanId,
|
|
172
|
+
parentSpanId,
|
|
173
|
+
startedAt: trace.startedAt,
|
|
174
|
+
endedAt: trace.endedAt,
|
|
175
|
+
status: status(trace.outcome),
|
|
176
|
+
};
|
|
177
|
+
if (child.kind === 'provider') {
|
|
178
|
+
return {
|
|
179
|
+
...common,
|
|
180
|
+
name: mode === 'agent' ? 'agent.provider_call' : spanName('chat', child.trace.modelId),
|
|
181
|
+
kind: mode === 'agent' ? 'internal' : 'client',
|
|
182
|
+
attributes: withErrorType(providerAttributes(child.trace, mode), trace.outcome, mode),
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
return {
|
|
186
|
+
...common,
|
|
187
|
+
name: mode === 'agent' ? 'agent.tool_body' : spanName('execute_tool', label(child.trace.toolName)),
|
|
188
|
+
kind: 'internal',
|
|
189
|
+
attributes: withErrorType(toolAttributes(child.trace, mode), trace.outcome, mode),
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Map one live prompt trace batch onto the spans an OpenTelemetry exporter records: the prompt root,
|
|
195
|
+
* then one child per provider call and tool body, in batch order. Permission decisions get no span.
|
|
196
|
+
* The result carries only what the batch carries, so it stays content-free.
|
|
197
|
+
*/
|
|
198
|
+
export function projectLivePromptTrace(
|
|
199
|
+
batch: ILivePromptTraceBatch, options: IOtelProjectionOptions,
|
|
200
|
+
): IOtelTraceProjection {
|
|
201
|
+
const { mode } = options;
|
|
202
|
+
const root: IOtelSpanProjection = {
|
|
203
|
+
name: mode === 'agent' ? 'agent.prompt_execution' : spanName('invoke_agent', options.agentName),
|
|
204
|
+
kind: 'internal',
|
|
205
|
+
traceId: batch.root.traceId,
|
|
206
|
+
spanId: batch.root.spanId,
|
|
207
|
+
startedAt: batch.root.startedAt,
|
|
208
|
+
endedAt: batch.root.endedAt,
|
|
209
|
+
status: status(batch.root.outcome),
|
|
210
|
+
attributes: withErrorType(rootAttributes(batch, options), batch.root.outcome, mode),
|
|
211
|
+
};
|
|
212
|
+
// Every child belongs to the root's trace, whatever trace ID an unchecked batch gave it.
|
|
213
|
+
const children = batch.children.filter(hasSpan)
|
|
214
|
+
.map((child) => projectTraceChild(child, root.traceId, root.spanId, mode));
|
|
215
|
+
return { ...(mode === 'agent' ? {} : { schemaUrl: GEN_AI_SCHEMA_URL }), root, children };
|
|
216
|
+
}
|