@sensu-ai/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +31 -0
- package/src/client.ts +434 -0
- package/src/index.ts +9 -0
- package/src/integrations/langchain.ts +175 -0
- package/src/integrations/openai.ts +109 -0
- package/src/types.ts +65 -0
- package/tsconfig.json +11 -0
package/package.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@sensu-ai/sdk",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"main": "./src/index.ts",
|
|
6
|
+
"exports": {
|
|
7
|
+
".": "./src/index.ts",
|
|
8
|
+
"./integrations/langchain": "./src/integrations/langchain.ts",
|
|
9
|
+
"./integrations/openai": "./src/integrations/openai.ts"
|
|
10
|
+
},
|
|
11
|
+
"scripts": {
|
|
12
|
+
"build": "tsc",
|
|
13
|
+
"dev": "tsc --watch"
|
|
14
|
+
},
|
|
15
|
+
"dependencies": {
|
|
16
|
+
"@sensu-ai/shared": "*",
|
|
17
|
+
"zod": "^3.23.8"
|
|
18
|
+
},
|
|
19
|
+
"devDependencies": {
|
|
20
|
+
"@types/node": "^20.0.0",
|
|
21
|
+
"typescript": "*"
|
|
22
|
+
},
|
|
23
|
+
"peerDependencies": {
|
|
24
|
+
"langchain": ">=0.1.0",
|
|
25
|
+
"openai": ">=4.0.0"
|
|
26
|
+
},
|
|
27
|
+
"peerDependenciesMeta": {
|
|
28
|
+
"langchain": { "optional": true },
|
|
29
|
+
"openai": { "optional": true }
|
|
30
|
+
}
|
|
31
|
+
}
|
package/src/client.ts
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
import { randomUUID } from 'crypto';
|
|
2
|
+
import type { TelemetryEvent } from '@sensu-ai/shared';
|
|
3
|
+
import type {
|
|
4
|
+
SensuClientOptions,
|
|
5
|
+
StartRunOptions,
|
|
6
|
+
StartStepOptions,
|
|
7
|
+
TrackLlmOptions,
|
|
8
|
+
TrackToolOptions,
|
|
9
|
+
RawLlmCallOptions,
|
|
10
|
+
} from './types.js';
|
|
11
|
+
|
|
12
|
+
// ---------------------------------------------------------------------------
|
|
13
|
+
// StepHandle — fluent API for a single step
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
|
|
16
|
+
export class StepHandle {
|
|
17
|
+
private readonly client: SensuClient;
|
|
18
|
+
readonly stepId: string;
|
|
19
|
+
readonly runId: string;
|
|
20
|
+
readonly sessionId: string;
|
|
21
|
+
readonly agentId: string;
|
|
22
|
+
readonly orgId: string;
|
|
23
|
+
readonly traceId: string;
|
|
24
|
+
readonly spanId: string;
|
|
25
|
+
private sequence: number;
|
|
26
|
+
private stepName?: string;
|
|
27
|
+
private ended = false;
|
|
28
|
+
|
|
29
|
+
constructor(
|
|
30
|
+
client: SensuClient,
|
|
31
|
+
opts: {
|
|
32
|
+
stepId: string;
|
|
33
|
+
runId: string;
|
|
34
|
+
sessionId: string;
|
|
35
|
+
agentId: string;
|
|
36
|
+
orgId: string;
|
|
37
|
+
traceId: string;
|
|
38
|
+
spanId: string;
|
|
39
|
+
sequence: number;
|
|
40
|
+
name?: string;
|
|
41
|
+
},
|
|
42
|
+
) {
|
|
43
|
+
this.client = client;
|
|
44
|
+
this.stepId = opts.stepId;
|
|
45
|
+
this.runId = opts.runId;
|
|
46
|
+
this.sessionId = opts.sessionId;
|
|
47
|
+
this.agentId = opts.agentId;
|
|
48
|
+
this.orgId = opts.orgId;
|
|
49
|
+
this.traceId = opts.traceId;
|
|
50
|
+
this.spanId = opts.spanId;
|
|
51
|
+
this.sequence = opts.sequence;
|
|
52
|
+
this.stepName = opts.name;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
private base() {
|
|
56
|
+
return {
|
|
57
|
+
event_id: randomUUID(),
|
|
58
|
+
timestamp: new Date().toISOString(),
|
|
59
|
+
org_id: this.orgId,
|
|
60
|
+
agent_id: this.agentId,
|
|
61
|
+
session_id: this.sessionId,
|
|
62
|
+
run_id: this.runId,
|
|
63
|
+
step_id: this.stepId,
|
|
64
|
+
trace_id: this.traceId,
|
|
65
|
+
span_id: randomUUID(),
|
|
66
|
+
parent_span_id: this.spanId,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Track an LLM call — wraps fn(), measures latency, emits event */
|
|
71
|
+
async trackLlm(opts: TrackLlmOptions): Promise<unknown> {
|
|
72
|
+
const startMs = Date.now();
|
|
73
|
+
const spanId = randomUUID();
|
|
74
|
+
|
|
75
|
+
this.client.enqueue({
|
|
76
|
+
...this.base(),
|
|
77
|
+
span_id: spanId,
|
|
78
|
+
event_type: 'llm.request.started',
|
|
79
|
+
provider: opts.provider,
|
|
80
|
+
model: opts.model,
|
|
81
|
+
max_context_tokens: opts.maxContextTokens,
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
let result: unknown;
|
|
85
|
+
let status: 'success' | 'error' = 'success';
|
|
86
|
+
let err: unknown;
|
|
87
|
+
|
|
88
|
+
try {
|
|
89
|
+
result = await opts.fn();
|
|
90
|
+
} catch (e) {
|
|
91
|
+
status = 'error';
|
|
92
|
+
err = e;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const latencyMs = Date.now() - startMs;
|
|
96
|
+
|
|
97
|
+
// Try to extract token usage from common response shapes
|
|
98
|
+
const usage = extractUsage(result, opts.model);
|
|
99
|
+
|
|
100
|
+
this.client.enqueue({
|
|
101
|
+
...this.base(),
|
|
102
|
+
span_id: spanId,
|
|
103
|
+
event_type: 'llm.request.completed',
|
|
104
|
+
provider: opts.provider,
|
|
105
|
+
model: opts.model,
|
|
106
|
+
max_context_tokens: opts.maxContextTokens,
|
|
107
|
+
latency_ms: latencyMs,
|
|
108
|
+
status,
|
|
109
|
+
...usage,
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
if (err) throw err;
|
|
113
|
+
return result;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Emit a raw LLM call event (when you have the stats already) */
|
|
117
|
+
recordLlm(opts: RawLlmCallOptions): void {
|
|
118
|
+
this.client.enqueue({
|
|
119
|
+
...this.base(),
|
|
120
|
+
event_type: 'llm.request.completed',
|
|
121
|
+
...opts,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Track a tool call — wraps fn(), measures latency */
|
|
126
|
+
async trackTool(opts: TrackToolOptions): Promise<unknown> {
|
|
127
|
+
const startMs = Date.now();
|
|
128
|
+
let result: unknown;
|
|
129
|
+
let status: 'success' | 'error' = 'success';
|
|
130
|
+
let err: unknown;
|
|
131
|
+
|
|
132
|
+
try {
|
|
133
|
+
result = await opts.fn();
|
|
134
|
+
} catch (e) {
|
|
135
|
+
status = 'error';
|
|
136
|
+
err = e;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const latencyMs = Date.now() - startMs;
|
|
140
|
+
const outputSize = estimateBytes(result);
|
|
141
|
+
|
|
142
|
+
this.client.enqueue({
|
|
143
|
+
...this.base(),
|
|
144
|
+
event_type: 'tool.call.completed',
|
|
145
|
+
tool_name: opts.toolName,
|
|
146
|
+
latency_ms: latencyMs,
|
|
147
|
+
status,
|
|
148
|
+
output_size_bytes: outputSize,
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
if (err) throw err;
|
|
152
|
+
return result;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
async end(): Promise<void> {
|
|
156
|
+
if (this.ended) return;
|
|
157
|
+
this.ended = true;
|
|
158
|
+
this.client.enqueue({
|
|
159
|
+
...this.base(),
|
|
160
|
+
event_type: 'agent.step.completed',
|
|
161
|
+
});
|
|
162
|
+
await this.client.flush();
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// ---------------------------------------------------------------------------
|
|
167
|
+
// RunHandle — fluent API for a single run
|
|
168
|
+
// ---------------------------------------------------------------------------
|
|
169
|
+
|
|
170
|
+
export class RunHandle {
|
|
171
|
+
private readonly client: SensuClient;
|
|
172
|
+
readonly runId: string;
|
|
173
|
+
readonly sessionId: string;
|
|
174
|
+
readonly agentId: string;
|
|
175
|
+
readonly orgId: string;
|
|
176
|
+
readonly traceId: string;
|
|
177
|
+
readonly spanId: string;
|
|
178
|
+
private stepCount = 0;
|
|
179
|
+
private ended = false;
|
|
180
|
+
|
|
181
|
+
constructor(
|
|
182
|
+
client: SensuClient,
|
|
183
|
+
opts: {
|
|
184
|
+
runId: string;
|
|
185
|
+
sessionId: string;
|
|
186
|
+
agentId: string;
|
|
187
|
+
orgId: string;
|
|
188
|
+
traceId: string;
|
|
189
|
+
spanId: string;
|
|
190
|
+
},
|
|
191
|
+
) {
|
|
192
|
+
this.client = client;
|
|
193
|
+
this.runId = opts.runId;
|
|
194
|
+
this.sessionId = opts.sessionId;
|
|
195
|
+
this.agentId = opts.agentId;
|
|
196
|
+
this.orgId = opts.orgId;
|
|
197
|
+
this.traceId = opts.traceId;
|
|
198
|
+
this.spanId = opts.spanId;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
private base() {
|
|
202
|
+
return {
|
|
203
|
+
event_id: randomUUID(),
|
|
204
|
+
timestamp: new Date().toISOString(),
|
|
205
|
+
org_id: this.orgId,
|
|
206
|
+
agent_id: this.agentId,
|
|
207
|
+
session_id: this.sessionId,
|
|
208
|
+
run_id: this.runId,
|
|
209
|
+
trace_id: this.traceId,
|
|
210
|
+
span_id: randomUUID(),
|
|
211
|
+
parent_span_id: this.spanId,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
startStep(opts: StartStepOptions = {}): StepHandle {
|
|
216
|
+
const stepId = opts.stepId ?? randomUUID();
|
|
217
|
+
const sequence = this.stepCount++;
|
|
218
|
+
|
|
219
|
+
this.client.enqueue({
|
|
220
|
+
...this.base(),
|
|
221
|
+
step_id: stepId,
|
|
222
|
+
event_type: 'agent.step.started',
|
|
223
|
+
step_type: opts.stepType ?? 'llm',
|
|
224
|
+
step_name: opts.name,
|
|
225
|
+
sequence: opts.sequence ?? sequence,
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
return new StepHandle(this.client, {
|
|
229
|
+
stepId,
|
|
230
|
+
runId: this.runId,
|
|
231
|
+
sessionId: this.sessionId,
|
|
232
|
+
agentId: this.agentId,
|
|
233
|
+
orgId: this.orgId,
|
|
234
|
+
traceId: this.traceId,
|
|
235
|
+
spanId: this.spanId,
|
|
236
|
+
sequence: opts.sequence ?? sequence,
|
|
237
|
+
name: opts.name,
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
async end(status: 'completed' | 'failed' = 'completed'): Promise<void> {
|
|
242
|
+
if (this.ended) return;
|
|
243
|
+
this.ended = true;
|
|
244
|
+
const eventType =
|
|
245
|
+
status === 'completed' ? 'agent.run.completed' : 'agent.run.failed';
|
|
246
|
+
this.client.enqueue({ ...this.base(), event_type: eventType });
|
|
247
|
+
await this.client.flush();
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// ---------------------------------------------------------------------------
|
|
252
|
+
// SensuClient
|
|
253
|
+
// ---------------------------------------------------------------------------
|
|
254
|
+
|
|
255
|
+
export class SensuClient {
|
|
256
|
+
private readonly apiKey: string;
|
|
257
|
+
private readonly baseUrl: string;
|
|
258
|
+
private readonly agentId: string;
|
|
259
|
+
private readonly orgId: string;
|
|
260
|
+
private readonly batchSize: number;
|
|
261
|
+
private readonly flushIntervalMs: number;
|
|
262
|
+
readonly disabled: boolean;
|
|
263
|
+
|
|
264
|
+
private buffer: TelemetryEvent[] = [];
|
|
265
|
+
private flushTimer: ReturnType<typeof setInterval> | null = null;
|
|
266
|
+
|
|
267
|
+
constructor(opts: SensuClientOptions = {}) {
|
|
268
|
+
const fromEnv = opts.fromEnv ?? false;
|
|
269
|
+
|
|
270
|
+
this.apiKey =
|
|
271
|
+
opts.apiKey ??
|
|
272
|
+
(fromEnv ? (process.env.SENSU_API_KEY ?? '') : '');
|
|
273
|
+
this.baseUrl =
|
|
274
|
+
opts.baseUrl ??
|
|
275
|
+
(fromEnv
|
|
276
|
+
? (process.env.SENSU_BASE_URL ?? 'http://localhost:3001')
|
|
277
|
+
: 'http://localhost:3001');
|
|
278
|
+
this.agentId =
|
|
279
|
+
opts.agentId ??
|
|
280
|
+
(fromEnv ? (process.env.SENSU_AGENT_ID ?? 'unknown-agent') : 'unknown-agent');
|
|
281
|
+
this.orgId =
|
|
282
|
+
opts.orgId ??
|
|
283
|
+
(fromEnv ? (process.env.SENSU_ORG_ID ?? '') : '');
|
|
284
|
+
|
|
285
|
+
this.batchSize = opts.batchSize ?? 10;
|
|
286
|
+
this.flushIntervalMs = opts.flushIntervalMs ?? 2000;
|
|
287
|
+
this.disabled = opts.disabled ?? false;
|
|
288
|
+
|
|
289
|
+
if (!this.disabled) {
|
|
290
|
+
this.flushTimer = setInterval(() => {
|
|
291
|
+
void this.flush();
|
|
292
|
+
}, this.flushIntervalMs);
|
|
293
|
+
if (this.flushTimer.unref) this.flushTimer.unref();
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** Enqueue an event for batched sending */
|
|
298
|
+
enqueue(event: TelemetryEvent): void {
|
|
299
|
+
if (this.disabled) return;
|
|
300
|
+
this.buffer.push(event);
|
|
301
|
+
if (this.buffer.length >= this.batchSize) {
|
|
302
|
+
void this.flush();
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/** Flush all buffered events to the Sensu API */
|
|
307
|
+
async flush(): Promise<void> {
|
|
308
|
+
if (this.disabled || this.buffer.length === 0) return;
|
|
309
|
+
const events = this.buffer.splice(0);
|
|
310
|
+
try {
|
|
311
|
+
const res = await fetch(`${this.baseUrl}/api/v1/events`, {
|
|
312
|
+
method: 'POST',
|
|
313
|
+
headers: {
|
|
314
|
+
'Content-Type': 'application/json',
|
|
315
|
+
'X-API-Key': this.apiKey,
|
|
316
|
+
},
|
|
317
|
+
body: JSON.stringify({ events }),
|
|
318
|
+
});
|
|
319
|
+
if (!res.ok) {
|
|
320
|
+
const body = await res.text();
|
|
321
|
+
console.error(`[sensu:sdk] flush failed ${res.status}: ${body}`);
|
|
322
|
+
}
|
|
323
|
+
} catch (err) {
|
|
324
|
+
// Re-queue on network error (best-effort)
|
|
325
|
+
console.error('[sensu:sdk] flush network error:', err);
|
|
326
|
+
this.buffer.unshift(...events);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/** Start a new agent run */
|
|
331
|
+
startRun(opts: StartRunOptions = {}): RunHandle {
|
|
332
|
+
const runId = opts.runId ?? randomUUID();
|
|
333
|
+
const sessionId = opts.sessionId ?? randomUUID();
|
|
334
|
+
const traceId = randomUUID();
|
|
335
|
+
const spanId = randomUUID();
|
|
336
|
+
|
|
337
|
+
this.enqueue({
|
|
338
|
+
event_id: randomUUID(),
|
|
339
|
+
event_type: 'agent.run.started',
|
|
340
|
+
timestamp: new Date().toISOString(),
|
|
341
|
+
org_id: this.orgId,
|
|
342
|
+
agent_id: this.agentId,
|
|
343
|
+
session_id: sessionId,
|
|
344
|
+
run_id: runId,
|
|
345
|
+
trace_id: traceId,
|
|
346
|
+
span_id: spanId,
|
|
347
|
+
run_type: opts.runType,
|
|
348
|
+
end_user_id: opts.endUserId,
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
return new RunHandle(this, {
|
|
352
|
+
runId,
|
|
353
|
+
sessionId,
|
|
354
|
+
agentId: this.agentId,
|
|
355
|
+
orgId: this.orgId,
|
|
356
|
+
traceId,
|
|
357
|
+
spanId,
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
destroy(): void {
|
|
362
|
+
if (this.flushTimer) clearInterval(this.flushTimer);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// ---------------------------------------------------------------------------
|
|
367
|
+
// Helpers
|
|
368
|
+
// ---------------------------------------------------------------------------
|
|
369
|
+
|
|
370
|
+
// Pricing per 1M tokens [input, output] in USD
|
|
371
|
+
const MODEL_PRICING: Record<string, [number, number]> = {
|
|
372
|
+
'claude-opus-4-6': [15.00, 75.00],
|
|
373
|
+
'claude-sonnet-4-6': [ 3.00, 15.00],
|
|
374
|
+
'claude-haiku-4-5-20251001': [ 0.80, 4.00],
|
|
375
|
+
'claude-3-5-sonnet-20241022': [ 3.00, 15.00],
|
|
376
|
+
'claude-3-5-haiku-20241022': [ 0.80, 4.00],
|
|
377
|
+
'claude-3-opus-20240229': [15.00, 75.00],
|
|
378
|
+
'gpt-4o': [ 5.00, 15.00],
|
|
379
|
+
'gpt-4o-mini': [ 0.15, 0.60],
|
|
380
|
+
'gpt-4-turbo': [10.00, 30.00],
|
|
381
|
+
};
|
|
382
|
+
|
|
383
|
+
function estimateCost(model: string, inputTokens: number, outputTokens: number): number {
|
|
384
|
+
const pricing = MODEL_PRICING[model];
|
|
385
|
+
if (!pricing) return 0;
|
|
386
|
+
const [inputPrice, outputPrice] = pricing;
|
|
387
|
+
return (inputTokens / 1_000_000) * inputPrice + (outputTokens / 1_000_000) * outputPrice;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
function extractUsage(result: unknown, model: string): Record<string, number | undefined> {
|
|
391
|
+
if (!result || typeof result !== 'object') return {};
|
|
392
|
+
const r = result as Record<string, unknown>;
|
|
393
|
+
|
|
394
|
+
// Anthropic shape: { usage: { input_tokens, output_tokens, cache_read_input_tokens } }
|
|
395
|
+
if (r['usage'] && typeof r['usage'] === 'object') {
|
|
396
|
+
const u = r['usage'] as Record<string, unknown>;
|
|
397
|
+
const inputTokens = num(u['input_tokens']) ?? 0;
|
|
398
|
+
const outputTokens = num(u['output_tokens']) ?? 0;
|
|
399
|
+
return {
|
|
400
|
+
input_tokens: inputTokens,
|
|
401
|
+
output_tokens: outputTokens,
|
|
402
|
+
cached_input_tokens: num(u['cache_read_input_tokens']),
|
|
403
|
+
total_tokens: inputTokens + outputTokens,
|
|
404
|
+
cost_usd_estimate: estimateCost(model, inputTokens, outputTokens),
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// OpenAI shape: { choices: [...], usage: { prompt_tokens, completion_tokens, total_tokens } }
|
|
409
|
+
if (r['choices'] && r['usage'] && typeof r['usage'] === 'object') {
|
|
410
|
+
const u = r['usage'] as Record<string, unknown>;
|
|
411
|
+
const inputTokens = num(u['prompt_tokens']) ?? 0;
|
|
412
|
+
const outputTokens = num(u['completion_tokens']) ?? 0;
|
|
413
|
+
return {
|
|
414
|
+
input_tokens: inputTokens,
|
|
415
|
+
output_tokens: outputTokens,
|
|
416
|
+
total_tokens: num(u['total_tokens']),
|
|
417
|
+
cost_usd_estimate: estimateCost(model, inputTokens, outputTokens),
|
|
418
|
+
};
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
return {};
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
function num(v: unknown): number | undefined {
|
|
425
|
+
return typeof v === 'number' ? v : undefined;
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
function estimateBytes(v: unknown): number {
|
|
429
|
+
try {
|
|
430
|
+
return JSON.stringify(v)?.length ?? 0;
|
|
431
|
+
} catch {
|
|
432
|
+
return 0;
|
|
433
|
+
}
|
|
434
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LangChain callback handler for Sensu telemetry.
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* import { SensuCallbackHandler } from '@sensu-ai/sdk/integrations/langchain';
|
|
6
|
+
* const handler = new SensuCallbackHandler({ client: sensu });
|
|
7
|
+
* const chain = new LLMChain({ ..., callbacks: [handler] });
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { randomUUID } from 'crypto';
|
|
11
|
+
import type { SensuClient } from '../client.ts';
|
|
12
|
+
|
|
13
|
+
interface LangChainCallbackHandlerOptions {
|
|
14
|
+
client: SensuClient;
|
|
15
|
+
sessionId?: string;
|
|
16
|
+
runId?: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
interface LangChainLlmStartInput {
|
|
20
|
+
name?: string;
|
|
21
|
+
[key: string]: unknown;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
interface LangChainLlmEndOutput {
|
|
25
|
+
generations?: Array<Array<{ text?: string; generationInfo?: Record<string, unknown> }>>;
|
|
26
|
+
llmOutput?: Record<string, unknown>;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export class SensuCallbackHandler {
|
|
30
|
+
private readonly client: SensuClient;
|
|
31
|
+
private readonly sessionId: string;
|
|
32
|
+
private runId: string;
|
|
33
|
+
private traceId: string;
|
|
34
|
+
private startTimes: Map<string, number> = new Map();
|
|
35
|
+
private toolStartTimes: Map<string, number> = new Map();
|
|
36
|
+
private stepIds: Map<string, string> = new Map();
|
|
37
|
+
|
|
38
|
+
constructor(opts: LangChainCallbackHandlerOptions) {
|
|
39
|
+
this.client = opts.client;
|
|
40
|
+
this.sessionId = opts.sessionId ?? randomUUID();
|
|
41
|
+
this.runId = opts.runId ?? randomUUID();
|
|
42
|
+
this.traceId = randomUUID();
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
private base(spanId?: string) {
|
|
46
|
+
return {
|
|
47
|
+
event_id: randomUUID(),
|
|
48
|
+
timestamp: new Date().toISOString(),
|
|
49
|
+
org_id: (this.client as unknown as { orgId: string }).orgId,
|
|
50
|
+
agent_id: (this.client as unknown as { agentId: string }).agentId,
|
|
51
|
+
session_id: this.sessionId,
|
|
52
|
+
run_id: this.runId,
|
|
53
|
+
trace_id: this.traceId,
|
|
54
|
+
span_id: spanId ?? randomUUID(),
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Called when a chain starts
|
|
59
|
+
async handleChainStart(_chain: unknown, _inputs: unknown, runId: string) {
|
|
60
|
+
const stepId = randomUUID();
|
|
61
|
+
this.stepIds.set(runId, stepId);
|
|
62
|
+
this.client.enqueue({
|
|
63
|
+
...this.base(randomUUID()),
|
|
64
|
+
step_id: stepId,
|
|
65
|
+
event_type: 'agent.step.started',
|
|
66
|
+
step_type: 'chain',
|
|
67
|
+
sequence: 0,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Called when a chain ends
|
|
72
|
+
async handleChainEnd(_outputs: unknown, runId: string) {
|
|
73
|
+
const stepId = this.stepIds.get(runId);
|
|
74
|
+
this.client.enqueue({
|
|
75
|
+
...this.base(),
|
|
76
|
+
step_id: stepId,
|
|
77
|
+
event_type: 'agent.step.completed',
|
|
78
|
+
});
|
|
79
|
+
this.stepIds.delete(runId);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Called when an LLM starts
|
|
83
|
+
async handleLLMStart(
|
|
84
|
+
llm: LangChainLlmStartInput,
|
|
85
|
+
_prompts: string[],
|
|
86
|
+
runId: string,
|
|
87
|
+
) {
|
|
88
|
+
this.startTimes.set(runId, Date.now());
|
|
89
|
+
this.client.enqueue({
|
|
90
|
+
...this.base(),
|
|
91
|
+
event_type: 'llm.request.started',
|
|
92
|
+
provider: 'langchain',
|
|
93
|
+
model: llm.name ?? 'unknown',
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Called when an LLM ends
|
|
98
|
+
async handleLLMEnd(output: LangChainLlmEndOutput, runId: string) {
|
|
99
|
+
const startMs = this.startTimes.get(runId);
|
|
100
|
+
const latencyMs = startMs ? Date.now() - startMs : undefined;
|
|
101
|
+
this.startTimes.delete(runId);
|
|
102
|
+
|
|
103
|
+
// Extract token usage from llmOutput if available
|
|
104
|
+
const usage = output.llmOutput?.['tokenUsage'] as Record<string, number> | undefined;
|
|
105
|
+
|
|
106
|
+
this.client.enqueue({
|
|
107
|
+
...this.base(),
|
|
108
|
+
event_type: 'llm.request.completed',
|
|
109
|
+
provider: 'langchain',
|
|
110
|
+
model: 'unknown',
|
|
111
|
+
latency_ms: latencyMs,
|
|
112
|
+
status: 'success',
|
|
113
|
+
input_tokens: usage?.['promptTokens'],
|
|
114
|
+
output_tokens: usage?.['completionTokens'],
|
|
115
|
+
total_tokens: usage?.['totalTokens'],
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Called when an LLM errors
|
|
120
|
+
async handleLLMError(err: Error, runId: string) {
|
|
121
|
+
const startMs = this.startTimes.get(runId);
|
|
122
|
+
const latencyMs = startMs ? Date.now() - startMs : undefined;
|
|
123
|
+
this.startTimes.delete(runId);
|
|
124
|
+
|
|
125
|
+
this.client.enqueue({
|
|
126
|
+
...this.base(),
|
|
127
|
+
event_type: 'llm.request.completed',
|
|
128
|
+
provider: 'langchain',
|
|
129
|
+
model: 'unknown',
|
|
130
|
+
latency_ms: latencyMs,
|
|
131
|
+
status: 'error',
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Called when a tool starts
|
|
136
|
+
async handleToolStart(_tool: unknown, _input: string, runId: string) {
|
|
137
|
+
this.toolStartTimes.set(runId, Date.now());
|
|
138
|
+
this.client.enqueue({
|
|
139
|
+
...this.base(),
|
|
140
|
+
event_type: 'tool.call.started',
|
|
141
|
+
tool_name: 'unknown',
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Called when a tool ends
|
|
146
|
+
async handleToolEnd(output: string, runId: string) {
|
|
147
|
+
const startMs = this.toolStartTimes.get(runId);
|
|
148
|
+
const latencyMs = startMs ? Date.now() - startMs : undefined;
|
|
149
|
+
this.toolStartTimes.delete(runId);
|
|
150
|
+
|
|
151
|
+
this.client.enqueue({
|
|
152
|
+
...this.base(),
|
|
153
|
+
event_type: 'tool.call.completed',
|
|
154
|
+
tool_name: 'unknown',
|
|
155
|
+
latency_ms: latencyMs,
|
|
156
|
+
status: 'success',
|
|
157
|
+
output_size_bytes: Buffer.byteLength(output ?? '', 'utf8'),
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// Called when a tool errors
|
|
162
|
+
async handleToolError(err: Error, runId: string) {
|
|
163
|
+
const startMs = this.toolStartTimes.get(runId);
|
|
164
|
+
const latencyMs = startMs ? Date.now() - startMs : undefined;
|
|
165
|
+
this.toolStartTimes.delete(runId);
|
|
166
|
+
|
|
167
|
+
this.client.enqueue({
|
|
168
|
+
...this.base(),
|
|
169
|
+
event_type: 'tool.call.completed',
|
|
170
|
+
tool_name: 'unknown',
|
|
171
|
+
latency_ms: latencyMs,
|
|
172
|
+
status: 'error',
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenAI SDK wrapper for Sensu telemetry.
|
|
3
|
+
* Wraps the OpenAI client to automatically track all completions.
|
|
4
|
+
*
|
|
5
|
+
* Usage:
|
|
6
|
+
* import { wrapOpenAI } from '@sensu-ai/sdk/integrations/openai';
|
|
7
|
+
* const openai = wrapOpenAI(new OpenAI({ apiKey }), { client: sensu, runHandle });
|
|
8
|
+
* const resp = await openai.chat.completions.create({ ... }); // auto-tracked
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { SensuClient, RunHandle } from '../client.ts';
|
|
12
|
+
|
|
13
|
+
interface OpenAILike {
|
|
14
|
+
chat: {
|
|
15
|
+
completions: {
|
|
16
|
+
create: (params: unknown) => Promise<unknown>;
|
|
17
|
+
};
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
interface WrapOpenAIOptions {
|
|
22
|
+
client: SensuClient;
|
|
23
|
+
runHandle?: RunHandle;
|
|
24
|
+
defaultModel?: string;
|
|
25
|
+
defaultProvider?: string;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
interface CompletionUsage {
|
|
29
|
+
prompt_tokens?: number;
|
|
30
|
+
completion_tokens?: number;
|
|
31
|
+
total_tokens?: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface CompletionResponse {
|
|
35
|
+
usage?: CompletionUsage;
|
|
36
|
+
model?: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function wrapOpenAI<T extends OpenAILike>(
|
|
40
|
+
openai: T,
|
|
41
|
+
opts: WrapOpenAIOptions,
|
|
42
|
+
): T {
|
|
43
|
+
const { client, runHandle } = opts;
|
|
44
|
+
|
|
45
|
+
const originalCreate = openai.chat.completions.create.bind(
|
|
46
|
+
openai.chat.completions,
|
|
47
|
+
);
|
|
48
|
+
|
|
49
|
+
openai.chat.completions.create = async (params: unknown): Promise<unknown> => {
|
|
50
|
+
const p = params as Record<string, unknown>;
|
|
51
|
+
const model = (p['model'] as string | undefined) ?? opts.defaultModel ?? 'unknown';
|
|
52
|
+
const provider = opts.defaultProvider ?? 'openai';
|
|
53
|
+
|
|
54
|
+
let step: import('../client.js').StepHandle | undefined;
|
|
55
|
+
if (runHandle) {
|
|
56
|
+
step = runHandle.startStep({ name: 'openai-completion', stepType: 'llm' });
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const startMs = Date.now();
|
|
60
|
+
let result: unknown;
|
|
61
|
+
let status: 'success' | 'error' = 'success';
|
|
62
|
+
let err: unknown;
|
|
63
|
+
|
|
64
|
+
try {
|
|
65
|
+
result = await originalCreate(params);
|
|
66
|
+
} catch (e) {
|
|
67
|
+
status = 'error';
|
|
68
|
+
err = e;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const latencyMs = Date.now() - startMs;
|
|
72
|
+
const r = result as CompletionResponse | undefined;
|
|
73
|
+
const usage = r?.usage;
|
|
74
|
+
|
|
75
|
+
const callOpts = {
|
|
76
|
+
provider,
|
|
77
|
+
model: r?.model ?? model,
|
|
78
|
+
input_tokens: usage?.prompt_tokens,
|
|
79
|
+
output_tokens: usage?.completion_tokens,
|
|
80
|
+
total_tokens: usage?.total_tokens,
|
|
81
|
+
latency_ms: latencyMs,
|
|
82
|
+
status,
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
if (step) {
|
|
86
|
+
step.recordLlm(callOpts);
|
|
87
|
+
void step.end();
|
|
88
|
+
} else {
|
|
89
|
+
// Emit standalone if no step context
|
|
90
|
+
client.enqueue({
|
|
91
|
+
event_id: crypto.randomUUID(),
|
|
92
|
+
event_type: 'llm.request.completed',
|
|
93
|
+
timestamp: new Date().toISOString(),
|
|
94
|
+
org_id: (client as unknown as { orgId: string }).orgId,
|
|
95
|
+
agent_id: (client as unknown as { agentId: string }).agentId,
|
|
96
|
+
session_id: crypto.randomUUID(),
|
|
97
|
+
run_id: crypto.randomUUID(),
|
|
98
|
+
trace_id: crypto.randomUUID(),
|
|
99
|
+
span_id: crypto.randomUUID(),
|
|
100
|
+
...callOpts,
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
if (err) throw err;
|
|
105
|
+
return result;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
return openai;
|
|
109
|
+
}
|
package/src/types.ts
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
export interface SensuClientOptions {
|
|
2
|
+
apiKey?: string;
|
|
3
|
+
baseUrl?: string;
|
|
4
|
+
agentId?: string;
|
|
5
|
+
orgId?: string;
|
|
6
|
+
/** Read config from environment variables */
|
|
7
|
+
fromEnv?: boolean;
|
|
8
|
+
/** Flush buffer when this many events accumulate */
|
|
9
|
+
batchSize?: number;
|
|
10
|
+
/** Flush buffer every N milliseconds */
|
|
11
|
+
flushIntervalMs?: number;
|
|
12
|
+
/** Disable all telemetry (useful for tests) */
|
|
13
|
+
disabled?: boolean;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface StartRunOptions {
|
|
17
|
+
sessionId?: string;
|
|
18
|
+
runType?: string;
|
|
19
|
+
endUserId?: string;
|
|
20
|
+
runId?: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface StartStepOptions {
|
|
24
|
+
name?: string;
|
|
25
|
+
stepType?: string;
|
|
26
|
+
sequence?: number;
|
|
27
|
+
stepId?: string;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface TrackLlmOptions {
|
|
31
|
+
provider: string;
|
|
32
|
+
model: string;
|
|
33
|
+
/** Wraps the LLM call and measures latency automatically */
|
|
34
|
+
fn: () => Promise<unknown>;
|
|
35
|
+
maxContextTokens?: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface LlmResult {
|
|
39
|
+
inputTokens?: number;
|
|
40
|
+
outputTokens?: number;
|
|
41
|
+
cachedInputTokens?: number;
|
|
42
|
+
totalTokens?: number;
|
|
43
|
+
costUsdEstimate?: number;
|
|
44
|
+
result: unknown;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface TrackToolOptions {
|
|
48
|
+
toolName: string;
|
|
49
|
+
fn: () => Promise<unknown>;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface RawLlmCallOptions {
|
|
53
|
+
provider: string;
|
|
54
|
+
model: string;
|
|
55
|
+
inputTokens?: number;
|
|
56
|
+
outputTokens?: number;
|
|
57
|
+
cachedInputTokens?: number;
|
|
58
|
+
totalTokens?: number;
|
|
59
|
+
maxContextTokens?: number;
|
|
60
|
+
contextUsedTokens?: number;
|
|
61
|
+
latencyMs?: number;
|
|
62
|
+
ttftMs?: number;
|
|
63
|
+
costUsdEstimate?: number;
|
|
64
|
+
status?: 'success' | 'error' | 'timeout';
|
|
65
|
+
}
|