observability-toolkit 1.8.0 → 1.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -3
- package/dist/backends/index.d.ts +76 -0
- package/dist/backends/index.d.ts.map +1 -1
- package/dist/backends/index.js +16 -0
- package/dist/backends/index.js.map +1 -1
- package/dist/backends/local-jsonl-boolean-search.test.js +1 -23
- package/dist/backends/local-jsonl-boolean-search.test.js.map +1 -1
- package/dist/backends/local-jsonl.d.ts +4 -1
- package/dist/backends/local-jsonl.d.ts.map +1 -1
- package/dist/backends/local-jsonl.js +185 -1
- package/dist/backends/local-jsonl.js.map +1 -1
- package/dist/backends/local-jsonl.test.js +434 -26
- package/dist/backends/local-jsonl.test.js.map +1 -1
- package/dist/backends/signoz-api.d.ts +32 -0
- package/dist/backends/signoz-api.d.ts.map +1 -1
- package/dist/backends/signoz-api.js +231 -33
- package/dist/backends/signoz-api.js.map +1 -1
- package/dist/backends/signoz-api.test.js +410 -63
- package/dist/backends/signoz-api.test.js.map +1 -1
- package/dist/lib/constants.d.ts +16 -0
- package/dist/lib/constants.d.ts.map +1 -1
- package/dist/lib/constants.js +121 -5
- package/dist/lib/constants.js.map +1 -1
- package/dist/lib/constants.test.js +202 -15
- package/dist/lib/constants.test.js.map +1 -1
- package/dist/lib/error-sanitizer.d.ts +57 -0
- package/dist/lib/error-sanitizer.d.ts.map +1 -0
- package/dist/lib/error-sanitizer.js +197 -0
- package/dist/lib/error-sanitizer.js.map +1 -0
- package/dist/lib/error-sanitizer.test.d.ts +8 -0
- package/dist/lib/error-sanitizer.test.d.ts.map +1 -0
- package/dist/lib/error-sanitizer.test.js +342 -0
- package/dist/lib/error-sanitizer.test.js.map +1 -0
- package/dist/lib/file-utils.d.ts +134 -0
- package/dist/lib/file-utils.d.ts.map +1 -1
- package/dist/lib/file-utils.js +395 -9
- package/dist/lib/file-utils.js.map +1 -1
- package/dist/lib/file-utils.test.js +444 -3
- package/dist/lib/file-utils.test.js.map +1 -1
- package/dist/lib/indexer.d.ts +9 -1
- package/dist/lib/indexer.d.ts.map +1 -1
- package/dist/lib/indexer.js +51 -2
- package/dist/lib/indexer.js.map +1 -1
- package/dist/lib/indexer.test.js +1 -16
- package/dist/lib/indexer.test.js.map +1 -1
- package/dist/lib/input-validator.d.ts +98 -0
- package/dist/lib/input-validator.d.ts.map +1 -0
- package/dist/lib/input-validator.js +245 -0
- package/dist/lib/input-validator.js.map +1 -0
- package/dist/lib/input-validator.test.d.ts +2 -0
- package/dist/lib/input-validator.test.d.ts.map +1 -0
- package/dist/lib/input-validator.test.js +287 -0
- package/dist/lib/input-validator.test.js.map +1 -0
- package/dist/lib/query-sanitizer.d.ts +95 -0
- package/dist/lib/query-sanitizer.d.ts.map +1 -0
- package/dist/lib/query-sanitizer.js +187 -0
- package/dist/lib/query-sanitizer.js.map +1 -0
- package/dist/lib/query-sanitizer.test.d.ts +5 -0
- package/dist/lib/query-sanitizer.test.d.ts.map +1 -0
- package/dist/lib/query-sanitizer.test.js +299 -0
- package/dist/lib/query-sanitizer.test.js.map +1 -0
- package/dist/server.d.ts +49 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +97 -13
- package/dist/server.js.map +1 -1
- package/dist/server.test.js +202 -0
- package/dist/server.test.js.map +1 -1
- package/dist/test-helpers/file-utils.d.ts +26 -0
- package/dist/test-helpers/file-utils.d.ts.map +1 -0
- package/dist/test-helpers/file-utils.js +43 -0
- package/dist/test-helpers/file-utils.js.map +1 -0
- package/dist/test-helpers/mock-backends.d.ts +28 -0
- package/dist/test-helpers/mock-backends.d.ts.map +1 -0
- package/dist/test-helpers/mock-backends.js +31 -0
- package/dist/test-helpers/mock-backends.js.map +1 -0
- package/dist/tools/index.d.ts +1 -0
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +1 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/query-evaluations.d.ts +183 -0
- package/dist/tools/query-evaluations.d.ts.map +1 -0
- package/dist/tools/query-evaluations.js +351 -0
- package/dist/tools/query-evaluations.js.map +1 -0
- package/dist/tools/query-evaluations.test.d.ts +5 -0
- package/dist/tools/query-evaluations.test.d.ts.map +1 -0
- package/dist/tools/query-evaluations.test.js +743 -0
- package/dist/tools/query-evaluations.test.js.map +1 -0
- package/dist/tools/query-llm-events.d.ts +9 -7
- package/dist/tools/query-llm-events.d.ts.map +1 -1
- package/dist/tools/query-llm-events.js +75 -48
- package/dist/tools/query-llm-events.js.map +1 -1
- package/dist/tools/query-logs.d.ts +26 -18
- package/dist/tools/query-logs.d.ts.map +1 -1
- package/dist/tools/query-logs.js +76 -52
- package/dist/tools/query-logs.js.map +1 -1
- package/dist/tools/query-logs.test.js +55 -73
- package/dist/tools/query-logs.test.js.map +1 -1
- package/dist/tools/query-metrics.d.ts +18 -18
- package/dist/tools/query-metrics.d.ts.map +1 -1
- package/dist/tools/query-metrics.js +101 -53
- package/dist/tools/query-metrics.js.map +1 -1
- package/dist/tools/query-metrics.test.js +14 -36
- package/dist/tools/query-metrics.test.js.map +1 -1
- package/dist/tools/query-traces.d.ts +24 -22
- package/dist/tools/query-traces.d.ts.map +1 -1
- package/dist/tools/query-traces.js +78 -53
- package/dist/tools/query-traces.js.map +1 -1
- package/dist/tools/query-traces.test.js +236 -36
- package/dist/tools/query-traces.test.js.map +1 -1
- package/package.json +1 -3
package/README.md
CHANGED
|
@@ -22,6 +22,7 @@ claude mcp add observability-toolkit -- node ~/.claude/mcp-servers/observability
|
|
|
22
22
|
| `obs_query_metrics` | Query metrics with aggregations (sum, avg, p50, p95, p99, rate) |
|
|
23
23
|
| `obs_query_logs` | Query logs with boolean search, field extraction |
|
|
24
24
|
| `obs_query_llm_events` | Query LLM events with token usage and duration metrics |
|
|
25
|
+
| `obs_query_evaluations` | Query evaluation events with aggregations and groupBy |
|
|
25
26
|
| `obs_health_check` | Check telemetry system health with cache statistics |
|
|
26
27
|
| `obs_context_stats` | Get context window utilization stats |
|
|
27
28
|
| `obs_get_trace_url` | Get SigNoz trace viewer URL (requires SigNoz) |
|
|
@@ -195,7 +196,7 @@ obs_health_check({ verbose: true })
|
|
|
195
196
|
| Exemplars | Metric-to-trace correlation |
|
|
196
197
|
| Aggregation Temporality | DELTA, CUMULATIVE support |
|
|
197
198
|
|
|
198
|
-
### OTel GenAI Semantic Conventions (
|
|
199
|
+
### OTel GenAI Semantic Conventions (10/10 compliance)
|
|
199
200
|
|
|
200
201
|
| Feature | Description |
|
|
201
202
|
|---------|-------------|
|
|
@@ -204,6 +205,10 @@ obs_health_check({ verbose: true })
|
|
|
204
205
|
| `gen_ai.conversation.id` | Filter LLM events by conversation ID |
|
|
205
206
|
| `gen_ai.agent.id/name` | Filter traces by agent attributes |
|
|
206
207
|
| `gen_ai.tool.name/call.id` | Filter traces by tool attributes |
|
|
208
|
+
| `gen_ai.response.model` | Actual model that responded |
|
|
209
|
+
| `gen_ai.response.finish_reasons` | Why generation stopped |
|
|
210
|
+
| `gen_ai.request.temperature` | Sampling temperature |
|
|
211
|
+
| `gen_ai.request.max_tokens` | Maximum output tokens |
|
|
207
212
|
| Percentiles | p50, p95, p99, rate aggregations |
|
|
208
213
|
|
|
209
214
|
### Performance
|
|
@@ -227,6 +232,17 @@ obs_health_check({ verbose: true })
|
|
|
227
232
|
| Circuit Breaker Logging | State transition visibility |
|
|
228
233
|
| Health Check Stats | Cache statistics in health output |
|
|
229
234
|
|
|
235
|
+
### Security
|
|
236
|
+
|
|
237
|
+
| Feature | Description |
|
|
238
|
+
|---------|-------------|
|
|
239
|
+
| Query Escaping | ClickHouse-specific escaping, 22-pattern blocklist |
|
|
240
|
+
| Memory Limits | MAX_RESULTS_IN_MEMORY=10000, streaming aggregation |
|
|
241
|
+
| Input Validation | limit≤1000, date range≤365 days, regex limits |
|
|
242
|
+
| Type Safety | NaN/Infinity rejection, explicit type assertions |
|
|
243
|
+
|
|
244
|
+
See [docs/security.md](docs/security.md) for details.
|
|
245
|
+
|
|
230
246
|
## Data Sources
|
|
231
247
|
|
|
232
248
|
### Local JSONL (Default)
|
|
@@ -269,11 +285,13 @@ const otlpMetrics = await backend.exportMetricsOTLP({ metricName: "http.duration
|
|
|
269
285
|
cd ~/.claude/mcp-servers/observability-toolkit
|
|
270
286
|
npm install
|
|
271
287
|
npm run build
|
|
272
|
-
npm test #
|
|
288
|
+
npm test # 1152 tests
|
|
273
289
|
npm run start
|
|
274
290
|
```
|
|
275
291
|
|
|
276
292
|
## Documentation
|
|
277
293
|
|
|
278
294
|
- [CHANGELOG.md](CHANGELOG.md) - Version history and changes
|
|
279
|
-
- [docs/
|
|
295
|
+
- [docs/security.md](docs/security.md) - Security controls and hardening
|
|
296
|
+
- [docs/ROADMAP.md](docs/ROADMAP.md) - Feature roadmap
|
|
297
|
+
- [docs/SESSION_HISTORY.md](docs/SESSION_HISTORY.md) - Development session logs
|
package/dist/backends/index.d.ts
CHANGED
|
@@ -244,6 +244,81 @@ export interface LLMEventQueryOptions extends QueryOptions {
|
|
|
244
244
|
/** Filter by conversation/session ID (gen_ai.conversation.id) */
|
|
245
245
|
conversationId?: string;
|
|
246
246
|
}
|
|
247
|
+
/**
|
|
248
|
+
* OTel GenAI evaluation attribute names
|
|
249
|
+
* https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-events/
|
|
250
|
+
*/
|
|
251
|
+
export declare const GENAI_EVALUATION_ATTRIBUTES: {
|
|
252
|
+
readonly NAME: "gen_ai.evaluation.name";
|
|
253
|
+
readonly SCORE_VALUE: "gen_ai.evaluation.score.value";
|
|
254
|
+
readonly SCORE_LABEL: "gen_ai.evaluation.score.label";
|
|
255
|
+
readonly EXPLANATION: "gen_ai.evaluation.explanation";
|
|
256
|
+
/** Custom extension: Score unit/scale (e.g., "ratio_0_1", "percentage") */
|
|
257
|
+
readonly SCORE_UNIT: "gen_ai.evaluation.score.unit";
|
|
258
|
+
/** Custom extension: Identity of the evaluator (model, human, system) */
|
|
259
|
+
readonly EVALUATOR: "gen_ai.evaluation.evaluator";
|
|
260
|
+
/** Custom extension: Type of evaluator */
|
|
261
|
+
readonly EVALUATOR_TYPE: "gen_ai.evaluation.evaluator.type";
|
|
262
|
+
};
|
|
263
|
+
/** Evaluator types for LLM quality evaluation */
|
|
264
|
+
export type EvaluatorType = 'llm' | 'human' | 'rule' | 'classifier';
|
|
265
|
+
/**
|
|
266
|
+
* Normalized evaluation result from gen_ai.evaluation.result events
|
|
267
|
+
*/
|
|
268
|
+
export interface EvaluationResult {
|
|
269
|
+
/** ISO timestamp of the evaluation */
|
|
270
|
+
timestamp: string;
|
|
271
|
+
/** Evaluation metric name (gen_ai.evaluation.name) - required */
|
|
272
|
+
evaluationName: string;
|
|
273
|
+
/** Numeric score (gen_ai.evaluation.score.value) */
|
|
274
|
+
scoreValue?: number;
|
|
275
|
+
/** Human-readable score label (gen_ai.evaluation.score.label) */
|
|
276
|
+
scoreLabel?: string;
|
|
277
|
+
/** Score unit/scale (e.g., "ratio_0_1", "percentage") */
|
|
278
|
+
scoreUnit?: string;
|
|
279
|
+
/** Free-form explanation (gen_ai.evaluation.explanation) */
|
|
280
|
+
explanation?: string;
|
|
281
|
+
/** Identity of the evaluator - model name, human ID, or system name */
|
|
282
|
+
evaluator?: string;
|
|
283
|
+
/** Type of evaluator (llm, human, rule, classifier) */
|
|
284
|
+
evaluatorType?: EvaluatorType;
|
|
285
|
+
/** Response ID for correlation (gen_ai.response.id) */
|
|
286
|
+
responseId?: string;
|
|
287
|
+
/** Trace ID for correlation */
|
|
288
|
+
traceId?: string;
|
|
289
|
+
/** Span ID for correlation */
|
|
290
|
+
spanId?: string;
|
|
291
|
+
/** Session ID for session-scoped queries */
|
|
292
|
+
sessionId?: string;
|
|
293
|
+
}
|
|
294
|
+
/** Aggregation functions for evaluation scores */
|
|
295
|
+
export type EvaluationAggregation = 'avg' | 'min' | 'max' | 'count' | 'p50' | 'p95' | 'p99';
|
|
296
|
+
/** Fields that can be used for groupBy in evaluation queries */
|
|
297
|
+
export type EvaluationGroupByField = 'evaluationName' | 'scoreLabel' | 'evaluator';
|
|
298
|
+
export interface EvaluationQueryOptions extends QueryOptions {
|
|
299
|
+
/** Filter by evaluation metric name (gen_ai.evaluation.name) */
|
|
300
|
+
evaluationName?: string;
|
|
301
|
+
/** Minimum score threshold */
|
|
302
|
+
scoreMin?: number;
|
|
303
|
+
/** Maximum score threshold */
|
|
304
|
+
scoreMax?: number;
|
|
305
|
+
/** Filter by score label (e.g., "pass", "fail", "relevant") */
|
|
306
|
+
scoreLabel?: string;
|
|
307
|
+
/** Filter by evaluator identity (exact match, e.g., "gpt-4-as-judge") */
|
|
308
|
+
evaluator?: string;
|
|
309
|
+
/** Filter by evaluator type (llm, human, rule, classifier) */
|
|
310
|
+
evaluatorType?: EvaluatorType;
|
|
311
|
+
/** Filter by response ID for correlation */
|
|
312
|
+
responseId?: string;
|
|
313
|
+
/** Filter by trace ID */
|
|
314
|
+
traceId?: string;
|
|
315
|
+
/** Filter by session ID */
|
|
316
|
+
sessionId?: string;
|
|
317
|
+
/** Aggregation function for scores */
|
|
318
|
+
aggregation?: EvaluationAggregation;
|
|
319
|
+
/** Group aggregations by these fields */
|
|
320
|
+
groupBy?: EvaluationGroupByField[];
|
|
321
|
+
}
|
|
247
322
|
export type { OTLPTraceExport, OTLPLogExport, OTLPMetricExport, } from '../lib/otlp-export.js';
|
|
248
323
|
export interface TelemetryBackend {
|
|
249
324
|
name: string;
|
|
@@ -251,6 +326,7 @@ export interface TelemetryBackend {
|
|
|
251
326
|
queryLogs(options: LogQueryOptions): Promise<LogRecord[]>;
|
|
252
327
|
queryMetrics(options: MetricQueryOptions): Promise<MetricDataPoint[]>;
|
|
253
328
|
queryLLMEvents?(options: LLMEventQueryOptions): Promise<LLMEvent[]>;
|
|
329
|
+
queryEvaluations?(options: EvaluationQueryOptions): Promise<EvaluationResult[]>;
|
|
254
330
|
healthCheck(): Promise<{
|
|
255
331
|
status: 'ok' | 'error';
|
|
256
332
|
message?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/backends/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;;GAGG;AACH,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,qEAAqE;IACrE,UAAU,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,CAAC;IAC1F,iFAAiF;IACjF,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IACnB,iFAAiF;IACjF,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;CAC7C;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,gFAAgF;IAChF,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,gDAAgD;IAChD,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,aAAa,GAAG,OAAO,GAAG,YAAY,CAAC;AAE5E,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC,SAAS,CAAC,EAAE,aAAa,CAAC;IAE1B,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAEvB,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;CACjD;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,CAAC;IAChC,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,CAAC;IAC7C,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,eAAO,MAAM,sBAAsB;;;;CAIzB,CAAC;AAEX,eAAO,MAAM,qBAAqB;;;;CAIxB,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,wBAAwB;;;;CAI3B,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,yBAAyB;;;CAG5B,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,sBAAsB;;;CAGzB,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,qBAAqB;;;;;CAKxB,CAAC;AAEX,MAAM,WAAW,iBAAkB,SAAQ,YAAY;IACrD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC;IAC5D,yDAAyD;IACzD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,sDAAsD;IACtD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,iDAAiD;IACjD,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,gDAAgD;IAChD,aAAa,CAAC,EAAE,sBAAsB,EAAE,CAAC;IAEzC,2CAA2C;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mDAAmD;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAED,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,qDAAqD;IACrD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gDAAgD;IAChD,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,gDAAgD;IAChD,aAAa,CAAC,EAAE,sBAAsB,EAAE,CAAC;IACzC,+CAA+C;IAC/C,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,0GAA0G;IAC1G,cAAc,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;IAC9B,gFAAgF;IAChF,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACtD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;IACvF,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,mEAAmE;IACnE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,MAAM,GAAG,YAAY,GAAG,cAAc,GAAG,cAAc,CAAC;AAEzF,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yFAAyF;IACzF,aAAa,CAAC,EAAE,kBAAkB,CAAC;IACnC,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAGD,YAAY,EACV,eAAe,EACf,aAAa,EACb,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IAEb,WAAW,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC9D,SAAS,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC1D,YAAY,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IACtE,cAAc,CAAC,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACpE,WAAW,IAAI,OAAO,CAAC;QAAE,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAGrE,aAAa,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,iBAAiB,EAAE,UAAU,CAAC,CAAC;IAGvE,gBAAgB,CAAC,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,uBAAuB,EAAE,eAAe,CAAC,CAAC;IACxG,cAAc,CAAC,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,OAAO,uBAAuB,EAAE,aAAa,CAAC,CAAC;IAClG,iBAAiB,CAAC,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,OAAO,uBAAuB,EAAE,gBAAgB,CAAC,CAAC;CAC5G"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/backends/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;;GAGG;AACH,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,qEAAqE;IACrE,UAAU,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,CAAC;IAC1F,iFAAiF;IACjF,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IACnB,iFAAiF;IACjF,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;CAC7C;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,gFAAgF;IAChF,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,gDAAgD;IAChD,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,aAAa,GAAG,OAAO,GAAG,YAAY,CAAC;AAE5E,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC,SAAS,CAAC,EAAE,aAAa,CAAC;IAE1B,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAEvB,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;CACjD;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,CAAC;IAChC,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,CAAC;IAC7C,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,eAAO,MAAM,sBAAsB;;;;CAIzB,CAAC;AAEX,eAAO,MAAM,qBAAqB;;;;CAIxB,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,wBAAwB;;;;CAI3B,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,yBAAyB;;;CAG5B,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,sBAAsB;;;CAGzB,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,qBAAqB;;;;;CAKxB,CAAC;AAEX,MAAM,WAAW,iBAAkB,SAAQ,YAAY;IACrD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC;IAC5D,yDAAyD;IACzD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,sDAAsD;IACtD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,iDAAiD;IACjD,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,gDAAgD;IAChD,aAAa,CAAC,EAAE,sBAAsB,EAAE,CAAC;IAEzC,2CAA2C;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mDAAmD;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAED,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,qDAAqD;IACrD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gDAAgD;IAChD,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,gDAAgD;IAChD,aAAa,CAAC,EAAE,sBAAsB,EAAE,CAAC;IACzC,+CAA+C;IAC/C,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,0GAA0G;IAC1G,cAAc,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;IAC9B,gFAAgF;IAChF,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACtD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;IACvF,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,mEAAmE;IACnE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,MAAM,GAAG,YAAY,GAAG,cAAc,GAAG,cAAc,CAAC;AAEzF,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yFAAyF;IACzF,aAAa,CAAC,EAAE,kBAAkB,CAAC;IACnC,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,eAAO,MAAM,2BAA2B;;;;;IAKtC,2EAA2E;;IAE3E,yEAAyE;;IAEzE,0CAA0C;;CAElC,CAAC;AAEX,iDAAiD;AACjD,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,YAAY,CAAC;AAEpE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,sCAAsC;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,cAAc,EAAE,MAAM,CAAC;IACvB,oDAAoD;IACpD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uDAAuD;IACvD,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,uDAAuD;IACvD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,kDAAkD;AAClD,MAAM,MAAM,qBAAqB,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;AAE5F,gEAAgE;AAChE,MAAM,MAAM,sBAAsB,GAAG,gBAAgB,GAAG,YAAY,GAAG,WAAW,CAAC;AAEnF,MAAM,WAAW,sBAAuB,SAAQ,YAAY;IAC1D,gEAAgE;IAChE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+DAA+D;IAC/D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,4CAA4C;IAC5C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yBAAyB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sCAAsC;IACtC,WAAW,CAAC,EAAE,qBAAqB,CAAC;IACpC,yCAAyC;IACzC,OAAO,CAAC,EAAE,sBAAsB,EAAE,CAAC;CACpC;AAGD,YAAY,EACV,eAAe,EACf,aAAa,EACb,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IAEb,WAAW,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC9D,SAAS,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC1D,YAAY,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IACtE,cAAc,CAAC,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACpE,gBAAgB,CAAC,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAChF,WAAW,IAAI,OAAO,CAAC;QAAE,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAGrE,aAAa,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,iBAAiB,EAAE,UAAU,CAAC,CAAC;IAGvE,gBAAgB,CAAC,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,uBAAuB,EAAE,eAAe,CAAC,CAAC;IACxG,cAAc,CAAC,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,OAAO,uBAAuB,EAAE,aAAa,CAAC,CAAC;IAClG,iBAAiB,CAAC,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,OAAO,uBAAuB,EAAE,gBAAgB,CAAC,CAAC;CAC5G"}
|
package/dist/backends/index.js
CHANGED
|
@@ -50,4 +50,20 @@ export const GENAI_CORE_ATTRIBUTES = {
|
|
|
50
50
|
SYSTEM: 'gen_ai.system',
|
|
51
51
|
CONVERSATION_ID: 'gen_ai.conversation.id',
|
|
52
52
|
};
|
|
53
|
+
/**
|
|
54
|
+
* OTel GenAI evaluation attribute names
|
|
55
|
+
* https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-events/
|
|
56
|
+
*/
|
|
57
|
+
export const GENAI_EVALUATION_ATTRIBUTES = {
|
|
58
|
+
NAME: 'gen_ai.evaluation.name',
|
|
59
|
+
SCORE_VALUE: 'gen_ai.evaluation.score.value',
|
|
60
|
+
SCORE_LABEL: 'gen_ai.evaluation.score.label',
|
|
61
|
+
EXPLANATION: 'gen_ai.evaluation.explanation',
|
|
62
|
+
/** Custom extension: Score unit/scale (e.g., "ratio_0_1", "percentage") */
|
|
63
|
+
SCORE_UNIT: 'gen_ai.evaluation.score.unit',
|
|
64
|
+
/** Custom extension: Identity of the evaluator (model, human, system) */
|
|
65
|
+
EVALUATOR: 'gen_ai.evaluation.evaluator',
|
|
66
|
+
/** Custom extension: Type of evaluator */
|
|
67
|
+
EVALUATOR_TYPE: 'gen_ai.evaluation.evaluator.type',
|
|
68
|
+
};
|
|
53
69
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/backends/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAgIH;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,QAAQ,EAAE,iBAAiB;IAC3B,UAAU,EAAE,mBAAmB;IAC/B,iBAAiB,EAAE,0BAA0B;CACrC,CAAC;AAEX,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,SAAS,EAAE,kBAAkB;IAC7B,YAAY,EAAE,qBAAqB;IACnC,SAAS,EAAE,kBAAkB;CACrB,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,KAAK,EAAE,sBAAsB;IAC7B,WAAW,EAAE,4BAA4B;IACzC,UAAU,EAAE,2BAA2B;CAC/B,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,KAAK,EAAE,uBAAuB;IAC9B,cAAc,EAAE,gCAAgC;CACxC,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,YAAY,EAAE,2BAA2B;IACzC,aAAa,EAAE,4BAA4B;CACnC,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,cAAc,EAAE,uBAAuB;IACvC,aAAa,EAAE,sBAAsB;IACrC,MAAM,EAAE,eAAe;IACvB,eAAe,EAAE,wBAAwB;CACjC,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/backends/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAgIH;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,QAAQ,EAAE,iBAAiB;IAC3B,UAAU,EAAE,mBAAmB;IAC/B,iBAAiB,EAAE,0BAA0B;CACrC,CAAC;AAEX,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,SAAS,EAAE,kBAAkB;IAC7B,YAAY,EAAE,qBAAqB;IACnC,SAAS,EAAE,kBAAkB;CACrB,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,KAAK,EAAE,sBAAsB;IAC7B,WAAW,EAAE,4BAA4B;IACzC,UAAU,EAAE,2BAA2B;CAC/B,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,KAAK,EAAE,uBAAuB;IAC9B,cAAc,EAAE,gCAAgC;CACxC,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,YAAY,EAAE,2BAA2B;IACzC,aAAa,EAAE,4BAA4B;CACnC,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,cAAc,EAAE,uBAAuB;IACvC,aAAa,EAAE,sBAAsB;IACrC,MAAM,EAAE,eAAe;IACvB,eAAe,EAAE,wBAAwB;CACjC,CAAC;AAqFX;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG;IACzC,IAAI,EAAE,wBAAwB;IAC9B,WAAW,EAAE,+BAA+B;IAC5C,WAAW,EAAE,+BAA+B;IAC5C,WAAW,EAAE,+BAA+B;IAC5C,2EAA2E;IAC3E,UAAU,EAAE,8BAA8B;IAC1C,yEAAyE;IACzE,SAAS,EAAE,6BAA6B;IACxC,0CAA0C;IAC1C,cAAc,EAAE,kCAAkC;CAC1C,CAAC"}
|
|
@@ -1,30 +1,8 @@
|
|
|
1
1
|
import { describe, it, beforeEach, afterEach } from 'node:test';
|
|
2
2
|
import * as assert from 'node:assert';
|
|
3
|
-
import * as fs from 'fs';
|
|
4
3
|
import * as path from 'path';
|
|
5
|
-
import * as os from 'os';
|
|
6
4
|
import { LocalJsonlBackend } from './local-jsonl.js';
|
|
7
|
-
|
|
8
|
-
* Test utilities for creating temp test fixtures
|
|
9
|
-
*/
|
|
10
|
-
function createTempDir() {
|
|
11
|
-
return fs.mkdtempSync(path.join(os.tmpdir(), 'local-jsonl-boolean-test-'));
|
|
12
|
-
}
|
|
13
|
-
function removeTempDir(dir) {
|
|
14
|
-
try {
|
|
15
|
-
fs.rmSync(dir, { recursive: true, force: true });
|
|
16
|
-
}
|
|
17
|
-
catch {
|
|
18
|
-
// Ignore cleanup errors
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
function writeJsonlFile(filePath, data) {
|
|
22
|
-
const content = data.map(item => JSON.stringify(item)).join('\n');
|
|
23
|
-
fs.writeFileSync(filePath, content, 'utf-8');
|
|
24
|
-
}
|
|
25
|
-
function getTestDate() {
|
|
26
|
-
return new Date().toISOString().split('T')[0];
|
|
27
|
-
}
|
|
5
|
+
import { createTempDir, removeTempDir, writeJsonlFile, getTestDate } from '../test-helpers/file-utils.js';
|
|
28
6
|
describe('LocalJsonlBackend Boolean Search', () => {
|
|
29
7
|
let tempDir;
|
|
30
8
|
let backend;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local-jsonl-boolean-search.test.js","sourceRoot":"","sources":["../../src/backends/local-jsonl-boolean-search.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAChE,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"local-jsonl-boolean-search.test.js","sourceRoot":"","sources":["../../src/backends/local-jsonl-boolean-search.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAChE,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAE1G,QAAQ,CAAC,kCAAkC,EAAE,GAAG,EAAE;IAChD,IAAI,OAAe,CAAC;IACpB,IAAI,OAA0B,CAAC;IAE/B,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,aAAa,EAAE,CAAC;QAC1B,OAAO,GAAG,IAAI,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,aAAa,CAAC,OAAO,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;QAC1C,EAAE,CAAC,yEAAyE,EAAE,KAAK,IAAI,EAAE;YACvF,MAAM,KAAK,GAAG,WAAW,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG;gBACf,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,0BAA0B,EAAE;gBACvE,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,uBAAuB,EAAE;gBACpE,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,0BAA0B,EAAE;gBACvE,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,oCAAoC,EAAE;aAClF,CAAC;YAEF,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,KAAK,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YAEpE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC;gBACtC,WAAW,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC;gBACtC,cAAc,EAAE,KAAK;aACtB,CAAC,CAAC;YAEH,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAC1B,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAC3C,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CACzC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sEAAsE,EAAE,KAAK,IAAI,EAAE;YACpF,MAAM,KAAK,GAAG,WAAW,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG;gBACf,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,0BAA0B,EAAE;gBACvE,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,uBAAuB,EAAE;gBACpE,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,sBAAsB,EAAE;gBACnE,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,kBAAkB,EAAE;aAChE,CAAC;YAEF,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,KAAK,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YAEpE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC;gBACtC,WAAW,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC;gBACtC,cAAc,EAAE,IAAI;aACrB,CAAC,CAAC;YAEH,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAC1B,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAC3C,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CACzC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;YACnF,MAAM,KAAK,GAAG,WAAW,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG;gBACf,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,kCAAkC,EAAE;gBAC/E,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,gBAAgB,EAAE;gBAC7D,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,2BAA2B,EAAE;gBACxE,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,8BAA8B,EAAE;aAC5E,CAAC;YAEF,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,KAAK,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YAEpE,mCAAmC;YACnC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC;gBACtC,WAAW,EAAE,CAAC,OAAO,EAAE,UAAU,CAAC;aACnC,CAAC,CAAC;YAEH,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAC1B,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACtC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAC1C,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;YACxE,MAAM,KAAK,GAAG,WAAW,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG;gBACf,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,0BAA0B,EAAE;gBACvE,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,0BAA0B,EAAE;gBACvE,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,8BAA8B,EAAE;aAC5E,CAAC;YAEF,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,KAAK,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YAEpE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC;gBACtC,WAAW,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC;gBAChC,cAAc,EAAE,KAAK;aACtB,CAAC,CAAC;YAEH,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,KAAK,GAAG,WAAW,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG;gBACf,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,eAAe,EAAE;gBAC5D,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,eAAe,EAAE;aAC7D,CAAC;YAEF,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,KAAK,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YAEpE,yCAAyC;YACzC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC;gBACtC,WAAW,EAAE,EAAE;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,KAAK,GAAG,WAAW,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG;gBACf,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,0BAA0B,EAAE;gBACvE,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,oBAAoB,EAAE;gBACjE,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,wBAAwB,EAAE;gBACrE,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,wBAAwB,EAAE;aACtE,CAAC;YAEF,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,KAAK,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YAEpE,kDAAkD;YAClD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC;gBACtC,MAAM,EAAE,YAAY;gBACpB,WAAW,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;gBACjC,cAAc,EAAE,IAAI;aACrB,CAAC,CAAC;YAEH,+DAA+D;YAC/D,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAC1B,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAC3C,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CACrF,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,KAAK,GAAG,WAAW,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG;gBACf,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,gBAAgB,EAAE;gBAC7D,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,0BAA0B,EAAE;gBACvE,EAAE,SAAS,EAAE,sBAAsB,EAAE,IAAI,EAAE,0BAA0B,EAAE;aACxE,CAAC;YAEF,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,KAAK,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;YAEpE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC;gBACtC,WAAW,EAAE,CAAC,OAAO,CAAC;gBACtB,cAAc,EAAE,KAAK;aACtB,CAAC,CAAC;YAEH,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* The local telemetry files use a flat JSONL format where each line is a complete
|
|
5
5
|
* span or log record, not the batched OpenTelemetry export format.
|
|
6
6
|
*/
|
|
7
|
-
import { TelemetryBackend, TraceSpan, LogRecord, MetricDataPoint, LLMEvent, TraceQueryOptions, LogQueryOptions, MetricQueryOptions, LLMEventQueryOptions, OTLPTraceExport, OTLPLogExport, OTLPMetricExport } from './index.js';
|
|
7
|
+
import { TelemetryBackend, TraceSpan, LogRecord, MetricDataPoint, LLMEvent, EvaluationResult, TraceQueryOptions, LogQueryOptions, MetricQueryOptions, LLMEventQueryOptions, EvaluationQueryOptions, OTLPTraceExport, OTLPLogExport, OTLPMetricExport } from './index.js';
|
|
8
8
|
export declare class LocalJsonlBackend implements TelemetryBackend {
|
|
9
9
|
name: string;
|
|
10
10
|
private telemetryDir;
|
|
@@ -12,6 +12,7 @@ export declare class LocalJsonlBackend implements TelemetryBackend {
|
|
|
12
12
|
private logCache;
|
|
13
13
|
private metricCache;
|
|
14
14
|
private llmEventCache;
|
|
15
|
+
private evaluationCache;
|
|
15
16
|
private useIndexes;
|
|
16
17
|
constructor(telemetryDir?: string, useIndexes?: boolean);
|
|
17
18
|
/**
|
|
@@ -38,6 +39,7 @@ export declare class LocalJsonlBackend implements TelemetryBackend {
|
|
|
38
39
|
*/
|
|
39
40
|
private calculateRate;
|
|
40
41
|
queryLLMEvents(options: LLMEventQueryOptions): Promise<LLMEvent[]>;
|
|
42
|
+
queryEvaluations(options: EvaluationQueryOptions): Promise<EvaluationResult[]>;
|
|
41
43
|
healthCheck(): Promise<{
|
|
42
44
|
status: 'ok' | 'error';
|
|
43
45
|
message?: string;
|
|
@@ -72,6 +74,7 @@ export declare class MultiDirectoryBackend implements TelemetryBackend {
|
|
|
72
74
|
queryLogs(options: LogQueryOptions): Promise<LogRecord[]>;
|
|
73
75
|
queryMetrics(options: MetricQueryOptions): Promise<MetricDataPoint[]>;
|
|
74
76
|
queryLLMEvents(options: LLMEventQueryOptions): Promise<LLMEvent[]>;
|
|
77
|
+
queryEvaluations(options: EvaluationQueryOptions): Promise<EvaluationResult[]>;
|
|
75
78
|
healthCheck(): Promise<{
|
|
76
79
|
status: 'ok' | 'error';
|
|
77
80
|
message?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local-jsonl.d.ts","sourceRoot":"","sources":["../../src/backends/local-jsonl.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EACL,gBAAgB,EAChB,SAAS,EACT,SAAS,EACT,eAAe,EACf,QAAQ,EACR,iBAAiB,EACjB,eAAe,EACf,kBAAkB,EAClB,oBAAoB,
|
|
1
|
+
{"version":3,"file":"local-jsonl.d.ts","sourceRoot":"","sources":["../../src/backends/local-jsonl.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EACL,gBAAgB,EAChB,SAAS,EACT,SAAS,EACT,eAAe,EACf,QAAQ,EACR,gBAAgB,EAEhB,iBAAiB,EACjB,eAAe,EACf,kBAAkB,EAClB,oBAAoB,EACpB,sBAAsB,EAOtB,eAAe,EACf,aAAa,EACb,gBAAgB,EAEjB,MAAM,YAAY,CAAC;AA4jBpB,qBAAa,iBAAkB,YAAW,gBAAgB;IACxD,IAAI,SAAiB;IACrB,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,UAAU,CAAiC;IACnD,OAAO,CAAC,QAAQ,CAAiC;IACjD,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,aAAa,CAAgC;IACrD,OAAO,CAAC,eAAe,CAAwC;IAC/D,OAAO,CAAC,UAAU,CAAU;gBAEhB,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,GAAE,OAAc;IAK7D;;OAEG;IACH,UAAU,IAAI,IAAI;IAQlB;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,iBAAiB,EAAE,UAAU,CAAC;IAUrE;;OAEG;IACH,OAAO,CAAC,WAAW;IAgBb,WAAW,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IA2I7D,SAAS,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAwHzD,YAAY,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAkF3E,OAAO,CAAC,SAAS;IA4FjB,OAAO,CAAC,mBAAmB;IAO3B;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAuBf,cAAc,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAsHlE,gBAAgB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAgH9E,WAAW,IAAI,OAAO,CAAC;QAAE,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAkC1E;;OAEG;IACG,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,eAAe,CAAC;IAK5E;;OAEG;IACG,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC;IAKtE;;OAEG;IACG,iBAAiB,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAIhF;AAED;;;GAGG;AACH,qBAAa,qBAAsB,YAAW,gBAAgB;IAC5D,IAAI,SAAqB;IACzB,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,WAAW,CAAsD;gBAE7D,GAAG,CAAC,EAAE,MAAM,EAAE,UAAU,GAAE,OAAc;IAKpD,cAAc,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAA;KAAE,CAAC;IAI/D,WAAW,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAwB7D,SAAS,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAwBzD,YAAY,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAarE,cAAc,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAwBlE,gBAAgB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAwB9E,WAAW,IAAI,OAAO,CAAC;QAAE,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAyBjJ;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,iBAAiB,EAAE,UAAU,CAAC;IAmCrE;;OAEG;IACG,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,eAAe,CAAC;IAK5E;;OAEG;IACG,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC;IAKtE;;OAEG;IACG,iBAAiB,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAIhF"}
|
|
@@ -5,11 +5,13 @@
|
|
|
5
5
|
* span or log record, not the batched OpenTelemetry export format.
|
|
6
6
|
*/
|
|
7
7
|
import { join } from 'path';
|
|
8
|
+
import { GENAI_EVALUATION_ATTRIBUTES, } from './index.js';
|
|
8
9
|
import { convertToOTLPTraces, convertToOTLPLogs, convertToOTLPMetrics, } from '../lib/otlp-export.js';
|
|
9
10
|
import { TELEMETRY_DIR, getTelemetryDirectories, getSpanKind, getStatusCodeName } from '../lib/constants.js';
|
|
10
11
|
import { listFiles, streamJsonl, parseDateFromFilename, getDateString, paginateResults, hasReachedLimit, } from '../lib/file-utils.js';
|
|
11
12
|
import { QueryCache, makeCacheKey } from '../lib/cache.js';
|
|
12
13
|
import { getIndexPath, readIndex, isIndexStale, queryIndex, readLinesByNumber, } from '../lib/indexer.js';
|
|
14
|
+
import { sanitizePath } from '../lib/error-sanitizer.js';
|
|
13
15
|
import { existsSync } from 'fs';
|
|
14
16
|
function startTiming() {
|
|
15
17
|
const start = performance.now();
|
|
@@ -147,6 +149,63 @@ function extractFieldsFromBody(body, fields) {
|
|
|
147
149
|
return undefined;
|
|
148
150
|
}
|
|
149
151
|
}
|
|
152
|
+
/**
|
|
153
|
+
* Validate and extract a non-empty trimmed string from attributes
|
|
154
|
+
*/
|
|
155
|
+
function extractValidString(value) {
|
|
156
|
+
if (typeof value !== 'string')
|
|
157
|
+
return undefined;
|
|
158
|
+
const trimmed = value.trim();
|
|
159
|
+
return trimmed.length > 0 ? trimmed : undefined;
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Validate and extract a finite number from attributes
|
|
163
|
+
*/
|
|
164
|
+
function extractValidNumber(value) {
|
|
165
|
+
if (typeof value !== 'number')
|
|
166
|
+
return undefined;
|
|
167
|
+
return Number.isFinite(value) ? value : undefined;
|
|
168
|
+
}
|
|
169
|
+
/** Valid evaluator types per our schema */
|
|
170
|
+
const VALID_EVALUATOR_TYPES = ['llm', 'human', 'rule', 'classifier'];
|
|
171
|
+
/**
|
|
172
|
+
* Extract and validate evaluatorType
|
|
173
|
+
*/
|
|
174
|
+
function extractValidEvaluatorType(value) {
|
|
175
|
+
if (typeof value !== 'string')
|
|
176
|
+
return undefined;
|
|
177
|
+
const trimmed = value.trim().toLowerCase();
|
|
178
|
+
return VALID_EVALUATOR_TYPES.includes(trimmed) ? trimmed : undefined;
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Convert flat evaluation to normalized EvaluationResult
|
|
182
|
+
* Includes runtime validation for data integrity (P0-2 fix)
|
|
183
|
+
*/
|
|
184
|
+
function normalizeEvaluation(raw) {
|
|
185
|
+
const attrs = raw.attributes;
|
|
186
|
+
// evaluationName is required per OTel spec - must be non-empty string
|
|
187
|
+
const evaluationName = extractValidString(attrs?.[GENAI_EVALUATION_ATTRIBUTES.NAME]);
|
|
188
|
+
if (!evaluationName || !raw.timestamp)
|
|
189
|
+
return null;
|
|
190
|
+
return {
|
|
191
|
+
timestamp: raw.timestamp,
|
|
192
|
+
evaluationName,
|
|
193
|
+
// Validate scoreValue is finite (rejects NaN, Infinity, -Infinity)
|
|
194
|
+
scoreValue: extractValidNumber(attrs?.[GENAI_EVALUATION_ATTRIBUTES.SCORE_VALUE]),
|
|
195
|
+
// Validate string fields are non-empty after trimming
|
|
196
|
+
scoreLabel: extractValidString(attrs?.[GENAI_EVALUATION_ATTRIBUTES.SCORE_LABEL]),
|
|
197
|
+
// Score unit for normalization (custom extension)
|
|
198
|
+
scoreUnit: extractValidString(attrs?.[GENAI_EVALUATION_ATTRIBUTES.SCORE_UNIT]),
|
|
199
|
+
explanation: extractValidString(attrs?.[GENAI_EVALUATION_ATTRIBUTES.EXPLANATION]),
|
|
200
|
+
// Evaluator fields (custom extension)
|
|
201
|
+
evaluator: extractValidString(attrs?.[GENAI_EVALUATION_ATTRIBUTES.EVALUATOR]),
|
|
202
|
+
evaluatorType: extractValidEvaluatorType(attrs?.[GENAI_EVALUATION_ATTRIBUTES.EVALUATOR_TYPE]),
|
|
203
|
+
responseId: extractValidString(attrs?.['gen_ai.response.id']),
|
|
204
|
+
traceId: raw.traceId,
|
|
205
|
+
spanId: raw.spanId,
|
|
206
|
+
sessionId: extractValidString(attrs?.['session.id']),
|
|
207
|
+
};
|
|
208
|
+
}
|
|
150
209
|
/**
|
|
151
210
|
* OTel-compliant severity number mapping
|
|
152
211
|
* https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
|
|
@@ -370,6 +429,7 @@ export class LocalJsonlBackend {
|
|
|
370
429
|
logCache = new QueryCache();
|
|
371
430
|
metricCache = new QueryCache();
|
|
372
431
|
llmEventCache = new QueryCache();
|
|
432
|
+
evaluationCache = new QueryCache();
|
|
373
433
|
useIndexes;
|
|
374
434
|
constructor(telemetryDir, useIndexes = true) {
|
|
375
435
|
this.telemetryDir = telemetryDir || TELEMETRY_DIR;
|
|
@@ -383,6 +443,7 @@ export class LocalJsonlBackend {
|
|
|
383
443
|
this.logCache.clear();
|
|
384
444
|
this.metricCache.clear();
|
|
385
445
|
this.llmEventCache.clear();
|
|
446
|
+
this.evaluationCache.clear();
|
|
386
447
|
}
|
|
387
448
|
/**
|
|
388
449
|
* Get cache statistics for all query caches
|
|
@@ -393,6 +454,7 @@ export class LocalJsonlBackend {
|
|
|
393
454
|
logs: this.logCache.getStats(),
|
|
394
455
|
metrics: this.metricCache.getStats(),
|
|
395
456
|
llmEvents: this.llmEventCache.getStats(),
|
|
457
|
+
evaluations: this.evaluationCache.getStats(),
|
|
396
458
|
};
|
|
397
459
|
}
|
|
398
460
|
/**
|
|
@@ -946,21 +1008,129 @@ export class LocalJsonlBackend {
|
|
|
946
1008
|
logTiming();
|
|
947
1009
|
return paginated;
|
|
948
1010
|
}
|
|
1011
|
+
async queryEvaluations(options) {
|
|
1012
|
+
const timer = startTiming();
|
|
1013
|
+
const logTiming = () => {
|
|
1014
|
+
const durationMs = timer.end();
|
|
1015
|
+
if (durationMs > SLOW_QUERY_THRESHOLD_MS) {
|
|
1016
|
+
console.warn(`[obs-toolkit] Slow query: queryEvaluations took ${durationMs.toFixed(1)}ms`);
|
|
1017
|
+
}
|
|
1018
|
+
};
|
|
1019
|
+
// Check cache first
|
|
1020
|
+
const cacheKey = makeCacheKey('evaluations', options);
|
|
1021
|
+
const cached = this.evaluationCache.get(cacheKey);
|
|
1022
|
+
if (cached) {
|
|
1023
|
+
logTiming();
|
|
1024
|
+
return cached;
|
|
1025
|
+
}
|
|
1026
|
+
const files = getFilesInRange(this.telemetryDir, /evaluations-\d{4}-\d{2}-\d{2}\.jsonl(\.gz)?$/, options.startDate, options.endDate);
|
|
1027
|
+
const results = [];
|
|
1028
|
+
const limit = options.limit || 100;
|
|
1029
|
+
const offset = options.offset || 0;
|
|
1030
|
+
// Build index query options for indexable filters
|
|
1031
|
+
const indexOptions = {
|
|
1032
|
+
traceId: options.traceId,
|
|
1033
|
+
evaluationName: options.evaluationName,
|
|
1034
|
+
scoreLabel: options.scoreLabel,
|
|
1035
|
+
responseId: options.responseId,
|
|
1036
|
+
evaluator: options.evaluator,
|
|
1037
|
+
};
|
|
1038
|
+
// Helper to apply non-indexable filters to an evaluation
|
|
1039
|
+
const applyFilters = (evaluation) => {
|
|
1040
|
+
// Score range filters - only apply when evaluation HAS a scoreValue (P1-1 fix)
|
|
1041
|
+
// Evaluations with only scoreLabel (qualitative) should pass through
|
|
1042
|
+
if (options.scoreMin !== undefined && evaluation.scoreValue !== undefined) {
|
|
1043
|
+
if (evaluation.scoreValue < options.scoreMin)
|
|
1044
|
+
return false;
|
|
1045
|
+
}
|
|
1046
|
+
if (options.scoreMax !== undefined && evaluation.scoreValue !== undefined) {
|
|
1047
|
+
if (evaluation.scoreValue > options.scoreMax)
|
|
1048
|
+
return false;
|
|
1049
|
+
}
|
|
1050
|
+
// Session filter
|
|
1051
|
+
if (options.sessionId && evaluation.sessionId !== options.sessionId) {
|
|
1052
|
+
return false;
|
|
1053
|
+
}
|
|
1054
|
+
// Evaluator type filter (not indexed, so always applied here)
|
|
1055
|
+
if (options.evaluatorType && evaluation.evaluatorType !== options.evaluatorType) {
|
|
1056
|
+
return false;
|
|
1057
|
+
}
|
|
1058
|
+
return true;
|
|
1059
|
+
};
|
|
1060
|
+
for (const file of files) {
|
|
1061
|
+
// Try to use index for pre-filtering
|
|
1062
|
+
const matchingLines = this.tryUseIndex(file, 'evaluations', indexOptions);
|
|
1063
|
+
if (matchingLines !== null) {
|
|
1064
|
+
// Use indexed query - read only matching lines
|
|
1065
|
+
const rawRecords = await readLinesByNumber(file, matchingLines);
|
|
1066
|
+
for (const raw of rawRecords) {
|
|
1067
|
+
const evaluation = normalizeEvaluation(raw);
|
|
1068
|
+
if (!evaluation)
|
|
1069
|
+
continue;
|
|
1070
|
+
if (!applyFilters(evaluation))
|
|
1071
|
+
continue;
|
|
1072
|
+
results.push(evaluation);
|
|
1073
|
+
if (hasReachedLimit(results.length, offset, limit)) {
|
|
1074
|
+
const paginated = paginateResults(results, offset, limit);
|
|
1075
|
+
this.evaluationCache.set(cacheKey, paginated);
|
|
1076
|
+
logTiming();
|
|
1077
|
+
return paginated;
|
|
1078
|
+
}
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
else {
|
|
1082
|
+
// Fall back to full file scan
|
|
1083
|
+
for await (const raw of streamJsonl(file)) {
|
|
1084
|
+
const evaluation = normalizeEvaluation(raw);
|
|
1085
|
+
if (!evaluation)
|
|
1086
|
+
continue;
|
|
1087
|
+
// Apply indexable filters (since no index was used)
|
|
1088
|
+
if (options.traceId && evaluation.traceId !== options.traceId)
|
|
1089
|
+
continue;
|
|
1090
|
+
if (options.evaluationName && !evaluation.evaluationName.includes(options.evaluationName))
|
|
1091
|
+
continue;
|
|
1092
|
+
if (options.scoreLabel && evaluation.scoreLabel !== options.scoreLabel)
|
|
1093
|
+
continue;
|
|
1094
|
+
if (options.responseId && evaluation.responseId !== options.responseId)
|
|
1095
|
+
continue;
|
|
1096
|
+
if (options.evaluator && evaluation.evaluator !== options.evaluator)
|
|
1097
|
+
continue;
|
|
1098
|
+
if (!applyFilters(evaluation))
|
|
1099
|
+
continue;
|
|
1100
|
+
results.push(evaluation);
|
|
1101
|
+
if (hasReachedLimit(results.length, offset, limit)) {
|
|
1102
|
+
const paginated = paginateResults(results, offset, limit);
|
|
1103
|
+
this.evaluationCache.set(cacheKey, paginated);
|
|
1104
|
+
logTiming();
|
|
1105
|
+
return paginated;
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
const paginated = paginateResults(results, offset, limit);
|
|
1111
|
+
this.evaluationCache.set(cacheKey, paginated);
|
|
1112
|
+
logTiming();
|
|
1113
|
+
return paginated;
|
|
1114
|
+
}
|
|
949
1115
|
async healthCheck() {
|
|
950
1116
|
if (!existsSync(this.telemetryDir)) {
|
|
951
|
-
|
|
1117
|
+
// Security: sanitize path to prevent information disclosure
|
|
1118
|
+
return { status: 'error', message: `Telemetry directory not found: ${sanitizePath(this.telemetryDir)}` };
|
|
952
1119
|
}
|
|
953
1120
|
const today = getDateString();
|
|
954
1121
|
const tracesFile = join(this.telemetryDir, `traces-${today}.jsonl`);
|
|
955
1122
|
const logsFile = join(this.telemetryDir, `logs-${today}.jsonl`);
|
|
956
1123
|
const llmEventsFile = join(this.telemetryDir, `llm-events-${today}.jsonl`);
|
|
1124
|
+
const evaluationsFile = join(this.telemetryDir, `evaluations-${today}.jsonl`);
|
|
957
1125
|
const hasTraces = existsSync(tracesFile);
|
|
958
1126
|
const hasLogs = existsSync(logsFile);
|
|
959
1127
|
const hasLLMEvents = existsSync(llmEventsFile);
|
|
1128
|
+
const hasEvaluations = existsSync(evaluationsFile);
|
|
960
1129
|
const found = [
|
|
961
1130
|
hasTraces ? 'traces' : '',
|
|
962
1131
|
hasLogs ? 'logs' : '',
|
|
963
1132
|
hasLLMEvents ? 'llm-events' : '',
|
|
1133
|
+
hasEvaluations ? 'evaluations' : '',
|
|
964
1134
|
].filter(Boolean).join(', ');
|
|
965
1135
|
if (!found) {
|
|
966
1136
|
return { status: 'ok', message: `No telemetry files for today (${today})` };
|
|
@@ -1054,6 +1224,19 @@ export class MultiDirectoryBackend {
|
|
|
1054
1224
|
}
|
|
1055
1225
|
return topResults;
|
|
1056
1226
|
}
|
|
1227
|
+
async queryEvaluations(options) {
|
|
1228
|
+
const limit = options.limit || 100;
|
|
1229
|
+
// Query all backends in parallel
|
|
1230
|
+
const allBackendResults = await Promise.all(this.backends.map(b => b.queryEvaluations({ ...options, limit })));
|
|
1231
|
+
// Merge results using bounded insertion for efficient top-K selection
|
|
1232
|
+
const topResults = [];
|
|
1233
|
+
for (const results of allBackendResults) {
|
|
1234
|
+
for (const evaluation of results) {
|
|
1235
|
+
insertSortedBounded(topResults, evaluation, limit, (a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
return topResults;
|
|
1239
|
+
}
|
|
1057
1240
|
async healthCheck() {
|
|
1058
1241
|
if (this.backends.length === 0) {
|
|
1059
1242
|
return { status: 'error', message: 'No telemetry directories found' };
|
|
@@ -1082,6 +1265,7 @@ export class MultiDirectoryBackend {
|
|
|
1082
1265
|
logs: { hits: 0, misses: 0, evictions: 0, size: 0 },
|
|
1083
1266
|
metrics: { hits: 0, misses: 0, evictions: 0, size: 0 },
|
|
1084
1267
|
llmEvents: { hits: 0, misses: 0, evictions: 0, size: 0 },
|
|
1268
|
+
evaluations: { hits: 0, misses: 0, evictions: 0, size: 0 },
|
|
1085
1269
|
};
|
|
1086
1270
|
for (const backend of this.backends) {
|
|
1087
1271
|
const stats = backend.getCacheStats();
|