observability-toolkit 1.8.5 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -5
- package/dist/backends/index.d.ts +163 -0
- package/dist/backends/index.d.ts.map +1 -1
- package/dist/backends/index.js +57 -0
- package/dist/backends/index.js.map +1 -1
- package/dist/backends/index.test.js +55 -1
- package/dist/backends/index.test.js.map +1 -1
- package/dist/backends/local-jsonl.d.ts +30 -0
- package/dist/backends/local-jsonl.d.ts.map +1 -1
- package/dist/backends/local-jsonl.js +912 -550
- package/dist/backends/local-jsonl.js.map +1 -1
- package/dist/backends/signoz-api-rate-limiter.test.js +2 -1
- package/dist/backends/signoz-api-rate-limiter.test.js.map +1 -1
- package/dist/backends/signoz-api.d.ts +16 -2
- package/dist/backends/signoz-api.d.ts.map +1 -1
- package/dist/backends/signoz-api.js +650 -534
- package/dist/backends/signoz-api.js.map +1 -1
- package/dist/backends/signoz-api.test.js +6 -5
- package/dist/backends/signoz-api.test.js.map +1 -1
- package/dist/lib/agent-as-judge.d.ts +388 -0
- package/dist/lib/agent-as-judge.d.ts.map +1 -0
- package/dist/lib/agent-as-judge.js +740 -0
- package/dist/lib/agent-as-judge.js.map +1 -0
- package/dist/lib/agent-as-judge.test.d.ts +5 -0
- package/dist/lib/agent-as-judge.test.d.ts.map +1 -0
- package/dist/lib/agent-as-judge.test.js +816 -0
- package/dist/lib/agent-as-judge.test.js.map +1 -0
- package/dist/lib/cache.d.ts +15 -2
- package/dist/lib/cache.d.ts.map +1 -1
- package/dist/lib/cache.js +16 -2
- package/dist/lib/cache.js.map +1 -1
- package/dist/lib/circuit-breaker.d.ts +18 -0
- package/dist/lib/circuit-breaker.d.ts.map +1 -1
- package/dist/lib/circuit-breaker.js +41 -8
- package/dist/lib/circuit-breaker.js.map +1 -1
- package/dist/lib/confident-export.d.ts +101 -0
- package/dist/lib/confident-export.d.ts.map +1 -0
- package/dist/lib/confident-export.js +393 -0
- package/dist/lib/confident-export.js.map +1 -0
- package/dist/lib/confident-export.test.d.ts +7 -0
- package/dist/lib/confident-export.test.d.ts.map +1 -0
- package/dist/lib/confident-export.test.js +835 -0
- package/dist/lib/confident-export.test.js.map +1 -0
- package/dist/lib/constants.d.ts +75 -0
- package/dist/lib/constants.d.ts.map +1 -1
- package/dist/lib/constants.js +104 -1
- package/dist/lib/constants.js.map +1 -1
- package/dist/lib/datadog-export.d.ts +156 -0
- package/dist/lib/datadog-export.d.ts.map +1 -0
- package/dist/lib/datadog-export.js +464 -0
- package/dist/lib/datadog-export.js.map +1 -0
- package/dist/lib/datadog-export.test.d.ts +14 -0
- package/dist/lib/datadog-export.test.d.ts.map +1 -0
- package/dist/lib/datadog-export.test.js +890 -0
- package/dist/lib/datadog-export.test.js.map +1 -0
- package/dist/lib/evaluation-hooks.d.ts +49 -0
- package/dist/lib/evaluation-hooks.d.ts.map +1 -0
- package/dist/lib/evaluation-hooks.js +488 -0
- package/dist/lib/evaluation-hooks.js.map +1 -0
- package/dist/lib/evaluation-hooks.test.d.ts +8 -0
- package/dist/lib/evaluation-hooks.test.d.ts.map +1 -0
- package/dist/lib/evaluation-hooks.test.js +624 -0
- package/dist/lib/evaluation-hooks.test.js.map +1 -0
- package/dist/lib/export-utils.d.ts +99 -0
- package/dist/lib/export-utils.d.ts.map +1 -0
- package/dist/lib/export-utils.js +238 -0
- package/dist/lib/export-utils.js.map +1 -0
- package/dist/lib/export-utils.test.d.ts +5 -0
- package/dist/lib/export-utils.test.d.ts.map +1 -0
- package/dist/lib/export-utils.test.js +193 -0
- package/dist/lib/export-utils.test.js.map +1 -0
- package/dist/lib/file-utils.d.ts +17 -2
- package/dist/lib/file-utils.d.ts.map +1 -1
- package/dist/lib/file-utils.js +24 -5
- package/dist/lib/file-utils.js.map +1 -1
- package/dist/lib/file-utils.test.js +30 -0
- package/dist/lib/file-utils.test.js.map +1 -1
- package/dist/lib/histogram.d.ts +119 -0
- package/dist/lib/histogram.d.ts.map +1 -0
- package/dist/lib/histogram.js +202 -0
- package/dist/lib/histogram.js.map +1 -0
- package/dist/lib/histogram.test.d.ts +5 -0
- package/dist/lib/histogram.test.d.ts.map +1 -0
- package/dist/lib/histogram.test.js +381 -0
- package/dist/lib/histogram.test.js.map +1 -0
- package/dist/lib/instrumentation.d.ts +153 -0
- package/dist/lib/instrumentation.d.ts.map +1 -0
- package/dist/lib/instrumentation.integration.test.d.ts +2 -0
- package/dist/lib/instrumentation.integration.test.d.ts.map +1 -0
- package/dist/lib/instrumentation.integration.test.js +589 -0
- package/dist/lib/instrumentation.integration.test.js.map +1 -0
- package/dist/lib/instrumentation.js +520 -0
- package/dist/lib/instrumentation.js.map +1 -0
- package/dist/lib/instrumentation.test.d.ts +2 -0
- package/dist/lib/instrumentation.test.d.ts.map +1 -0
- package/dist/lib/instrumentation.test.js +821 -0
- package/dist/lib/instrumentation.test.js.map +1 -0
- package/dist/lib/langfuse-export.d.ts +125 -0
- package/dist/lib/langfuse-export.d.ts.map +1 -0
- package/dist/lib/langfuse-export.js +367 -0
- package/dist/lib/langfuse-export.js.map +1 -0
- package/dist/lib/langfuse-export.test.d.ts +7 -0
- package/dist/lib/langfuse-export.test.d.ts.map +1 -0
- package/dist/lib/langfuse-export.test.js +1007 -0
- package/dist/lib/langfuse-export.test.js.map +1 -0
- package/dist/lib/llm-as-judge.d.ts +657 -0
- package/dist/lib/llm-as-judge.d.ts.map +1 -0
- package/dist/lib/llm-as-judge.js +1397 -0
- package/dist/lib/llm-as-judge.js.map +1 -0
- package/dist/lib/llm-as-judge.test.d.ts +2 -0
- package/dist/lib/llm-as-judge.test.d.ts.map +1 -0
- package/dist/lib/llm-as-judge.test.js +2409 -0
- package/dist/lib/llm-as-judge.test.js.map +1 -0
- package/dist/lib/logger.d.ts +1 -1
- package/dist/lib/logger.d.ts.map +1 -1
- package/dist/lib/logger.js.map +1 -1
- package/dist/lib/metrics.d.ts +62 -0
- package/dist/lib/metrics.d.ts.map +1 -0
- package/dist/lib/metrics.js +166 -0
- package/dist/lib/metrics.js.map +1 -0
- package/dist/lib/metrics.test.d.ts +5 -0
- package/dist/lib/metrics.test.d.ts.map +1 -0
- package/dist/lib/metrics.test.js +189 -0
- package/dist/lib/metrics.test.js.map +1 -0
- package/dist/lib/parse-stats.d.ts +119 -0
- package/dist/lib/parse-stats.d.ts.map +1 -0
- package/dist/lib/parse-stats.js +206 -0
- package/dist/lib/parse-stats.js.map +1 -0
- package/dist/lib/parse-stats.test.d.ts +5 -0
- package/dist/lib/parse-stats.test.d.ts.map +1 -0
- package/dist/lib/parse-stats.test.js +283 -0
- package/dist/lib/parse-stats.test.js.map +1 -0
- package/dist/lib/phoenix-export.d.ts +109 -0
- package/dist/lib/phoenix-export.d.ts.map +1 -0
- package/dist/lib/phoenix-export.js +429 -0
- package/dist/lib/phoenix-export.js.map +1 -0
- package/dist/lib/phoenix-export.test.d.ts +11 -0
- package/dist/lib/phoenix-export.test.d.ts.map +1 -0
- package/dist/lib/phoenix-export.test.js +725 -0
- package/dist/lib/phoenix-export.test.js.map +1 -0
- package/dist/lib/server-utils.d.ts +6 -1
- package/dist/lib/server-utils.d.ts.map +1 -1
- package/dist/lib/server-utils.js +9 -1
- package/dist/lib/server-utils.js.map +1 -1
- package/dist/lib/shared-schemas.d.ts +6 -0
- package/dist/lib/shared-schemas.d.ts.map +1 -1
- package/dist/lib/shared-schemas.js +11 -4
- package/dist/lib/shared-schemas.js.map +1 -1
- package/dist/lib/verification-events.d.ts +100 -0
- package/dist/lib/verification-events.d.ts.map +1 -0
- package/dist/lib/verification-events.js +162 -0
- package/dist/lib/verification-events.js.map +1 -0
- package/dist/lib/verification-events.test.d.ts +5 -0
- package/dist/lib/verification-events.test.d.ts.map +1 -0
- package/dist/lib/verification-events.test.js +193 -0
- package/dist/lib/verification-events.test.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +77 -21
- package/dist/server.js.map +1 -1
- package/dist/tools/context-stats.d.ts.map +1 -1
- package/dist/tools/context-stats.js +6 -8
- package/dist/tools/context-stats.js.map +1 -1
- package/dist/tools/export-confident.d.ts +145 -0
- package/dist/tools/export-confident.d.ts.map +1 -0
- package/dist/tools/export-confident.js +134 -0
- package/dist/tools/export-confident.js.map +1 -0
- package/dist/tools/export-confident.test.d.ts +7 -0
- package/dist/tools/export-confident.test.d.ts.map +1 -0
- package/dist/tools/export-confident.test.js +332 -0
- package/dist/tools/export-confident.test.js.map +1 -0
- package/dist/tools/export-datadog.d.ts +160 -0
- package/dist/tools/export-datadog.d.ts.map +1 -0
- package/dist/tools/export-datadog.js +160 -0
- package/dist/tools/export-datadog.js.map +1 -0
- package/dist/tools/export-datadog.test.d.ts +8 -0
- package/dist/tools/export-datadog.test.d.ts.map +1 -0
- package/dist/tools/export-datadog.test.js +419 -0
- package/dist/tools/export-datadog.test.js.map +1 -0
- package/dist/tools/export-langfuse.d.ts +137 -0
- package/dist/tools/export-langfuse.d.ts.map +1 -0
- package/dist/tools/export-langfuse.js +131 -0
- package/dist/tools/export-langfuse.js.map +1 -0
- package/dist/tools/export-langfuse.test.d.ts +7 -0
- package/dist/tools/export-langfuse.test.d.ts.map +1 -0
- package/dist/tools/export-langfuse.test.js +303 -0
- package/dist/tools/export-langfuse.test.js.map +1 -0
- package/dist/tools/export-phoenix.d.ts +145 -0
- package/dist/tools/export-phoenix.d.ts.map +1 -0
- package/dist/tools/export-phoenix.js +135 -0
- package/dist/tools/export-phoenix.js.map +1 -0
- package/dist/tools/export-phoenix.test.d.ts +7 -0
- package/dist/tools/export-phoenix.test.d.ts.map +1 -0
- package/dist/tools/export-phoenix.test.js +316 -0
- package/dist/tools/export-phoenix.test.js.map +1 -0
- package/dist/tools/health-check.d.ts +26 -0
- package/dist/tools/health-check.d.ts.map +1 -1
- package/dist/tools/health-check.js +36 -7
- package/dist/tools/health-check.js.map +1 -1
- package/dist/tools/index.d.ts +6 -0
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +6 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/inject-evaluations.d.ts +1315 -0
- package/dist/tools/inject-evaluations.d.ts.map +1 -0
- package/dist/tools/inject-evaluations.js +121 -0
- package/dist/tools/inject-evaluations.js.map +1 -0
- package/dist/tools/inject-evaluations.test.d.ts +5 -0
- package/dist/tools/inject-evaluations.test.d.ts.map +1 -0
- package/dist/tools/inject-evaluations.test.js +359 -0
- package/dist/tools/inject-evaluations.test.js.map +1 -0
- package/dist/tools/query-evaluations.d.ts +25 -4
- package/dist/tools/query-evaluations.d.ts.map +1 -1
- package/dist/tools/query-evaluations.js +10 -0
- package/dist/tools/query-evaluations.js.map +1 -1
- package/dist/tools/query-llm-events.js +2 -2
- package/dist/tools/query-llm-events.js.map +1 -1
- package/dist/tools/query-logs.d.ts +8 -8
- package/dist/tools/query-logs.js +3 -3
- package/dist/tools/query-logs.js.map +1 -1
- package/dist/tools/query-metrics.d.ts +4 -4
- package/dist/tools/query-metrics.js +2 -2
- package/dist/tools/query-metrics.js.map +1 -1
- package/dist/tools/query-traces.d.ts +8 -8
- package/dist/tools/query-verifications.d.ts +111 -0
- package/dist/tools/query-verifications.d.ts.map +1 -0
- package/dist/tools/query-verifications.js +101 -0
- package/dist/tools/query-verifications.js.map +1 -0
- package/dist/tools/query-verifications.test.d.ts +5 -0
- package/dist/tools/query-verifications.test.d.ts.map +1 -0
- package/dist/tools/query-verifications.test.js +156 -0
- package/dist/tools/query-verifications.test.js.map +1 -0
- package/dist/types/evaluation-hooks.d.ts +176 -0
- package/dist/types/evaluation-hooks.d.ts.map +1 -0
- package/dist/types/evaluation-hooks.js +49 -0
- package/dist/types/evaluation-hooks.js.map +1 -0
- package/package.json +10 -2
package/README.md
CHANGED
|
@@ -23,10 +23,12 @@ claude mcp add observability-toolkit -- node ~/.claude/mcp-servers/observability
|
|
|
23
23
|
| `obs_query_logs` | Query logs with boolean search, field extraction |
|
|
24
24
|
| `obs_query_llm_events` | Query LLM events with token usage and duration metrics |
|
|
25
25
|
| `obs_query_evaluations` | Query evaluation events with aggregations and groupBy |
|
|
26
|
+
| `obs_query_verifications` | Query human verification events for EU AI Act compliance |
|
|
26
27
|
| `obs_health_check` | Check telemetry system health with cache statistics |
|
|
27
28
|
| `obs_context_stats` | Get context window utilization stats |
|
|
28
29
|
| `obs_get_trace_url` | Get SigNoz trace viewer URL (requires SigNoz) |
|
|
29
30
|
| `obs_setup_claudeignore` | Add entries to .claudeignore |
|
|
31
|
+
| `obs_export_langfuse` | Export evaluations to Langfuse via OTLP HTTP |
|
|
30
32
|
|
|
31
33
|
## Configuration
|
|
32
34
|
|
|
@@ -37,6 +39,9 @@ claude mcp add observability-toolkit -- node ~/.claude/mcp-servers/observability
|
|
|
37
39
|
| `SIGNOZ_API_KEY` | SigNoz API key | - |
|
|
38
40
|
| `CACHE_TTL_MS` | Query cache TTL in milliseconds | `60000` |
|
|
39
41
|
| `RETENTION_DAYS` | Days to retain telemetry files | `7` |
|
|
42
|
+
| `LANGFUSE_ENDPOINT` | Langfuse OTLP endpoint URL | - |
|
|
43
|
+
| `LANGFUSE_PUBLIC_KEY` | Langfuse public key | - |
|
|
44
|
+
| `LANGFUSE_SECRET_KEY` | Langfuse secret key | - |
|
|
40
45
|
|
|
41
46
|
## Usage Examples
|
|
42
47
|
|
|
@@ -339,19 +344,135 @@ const otlpLogs = await backend.exportLogsOTLP({ severity: "ERROR" });
|
|
|
339
344
|
const otlpMetrics = await backend.exportMetricsOTLP({ metricName: "http.duration" });
|
|
340
345
|
```
|
|
341
346
|
|
|
347
|
+
### Langfuse Export (v1.8.6+)
|
|
348
|
+
|
|
349
|
+
Export evaluations to Langfuse for unified tracing and evaluation analysis:
|
|
350
|
+
|
|
351
|
+
```javascript
|
|
352
|
+
// Export all evaluations from last 7 days
|
|
353
|
+
obs_export_langfuse({})
|
|
354
|
+
|
|
355
|
+
// Export with filters
|
|
356
|
+
obs_export_langfuse({
|
|
357
|
+
evaluationName: "quality",
|
|
358
|
+
scoreMin: 0.8,
|
|
359
|
+
limit: 500,
|
|
360
|
+
batchSize: 100
|
|
361
|
+
})
|
|
362
|
+
|
|
363
|
+
// Dry run to preview export
|
|
364
|
+
obs_export_langfuse({
|
|
365
|
+
startDate: "2026-01-28",
|
|
366
|
+
dryRun: true
|
|
367
|
+
})
|
|
368
|
+
|
|
369
|
+
// Override credentials (for testing)
|
|
370
|
+
obs_export_langfuse({
|
|
371
|
+
endpoint: "https://cloud.langfuse.com",
|
|
372
|
+
publicKey: "pk-lf-...",
|
|
373
|
+
secretKey: "sk-lf-..."
|
|
374
|
+
})
|
|
375
|
+
```
|
|
376
|
+
|
|
377
|
+
Features:
|
|
378
|
+
- Batched OTLP HTTP export with retry logic
|
|
379
|
+
- Memory protection (400MB warn, 600MB abort)
|
|
380
|
+
- Progress logging for large exports
|
|
381
|
+
- Credential sanitization in error messages
|
|
382
|
+
- DNS rebinding protection
|
|
383
|
+
|
|
384
|
+
## Evaluation Libraries
|
|
385
|
+
|
|
386
|
+
### LLM-as-Judge (`src/lib/llm-as-judge.ts`)
|
|
387
|
+
|
|
388
|
+
Single-pass LLM evaluation for output quality:
|
|
389
|
+
|
|
390
|
+
```typescript
|
|
391
|
+
import { gEval, qagEvaluate, JudgeCircuitBreaker } from './lib/llm-as-judge.js';
|
|
392
|
+
|
|
393
|
+
// G-Eval pattern with chain-of-thought
|
|
394
|
+
const result = await gEval(testCase, criteria, llmFn);
|
|
395
|
+
|
|
396
|
+
// QAG faithfulness evaluation
|
|
397
|
+
const faithfulness = await qagEvaluate(testCase, llmFn);
|
|
398
|
+
|
|
399
|
+
// Production circuit breaker
|
|
400
|
+
const breaker = new JudgeCircuitBreaker(5, 60000);
|
|
401
|
+
const result = await breaker.evaluate(() => gEval(...));
|
|
402
|
+
```
|
|
403
|
+
|
|
404
|
+
### Agent-as-Judge (`src/lib/agent-as-judge.ts`)
|
|
405
|
+
|
|
406
|
+
Multi-step agent evaluation with trajectory analysis:
|
|
407
|
+
|
|
408
|
+
```typescript
|
|
409
|
+
import {
|
|
410
|
+
verifyToolCalls,
|
|
411
|
+
aggregateStepScores,
|
|
412
|
+
analyzeTrajectory,
|
|
413
|
+
collectiveConsensus,
|
|
414
|
+
ProceduralJudge,
|
|
415
|
+
ReactiveJudge,
|
|
416
|
+
} from './lib/agent-as-judge.js';
|
|
417
|
+
|
|
418
|
+
// Verify tool call correctness
|
|
419
|
+
const verifications = verifyToolCalls(actions, expectedTools);
|
|
420
|
+
|
|
421
|
+
// Analyze agent trajectory efficiency
|
|
422
|
+
const metrics = analyzeTrajectory({ actions, expectedSteps: 5 });
|
|
423
|
+
|
|
424
|
+
// Multi-agent consensus evaluation
|
|
425
|
+
const consensus = await collectiveConsensus(judges, { id: 'eval-1' }, {
|
|
426
|
+
rounds: 3,
|
|
427
|
+
convergenceThreshold: 0.05,
|
|
428
|
+
});
|
|
429
|
+
|
|
430
|
+
// Procedural multi-stage evaluation
|
|
431
|
+
const proceduralJudge = new ProceduralJudge([
|
|
432
|
+
{ name: 'syntax', evaluate: syntaxChecker },
|
|
433
|
+
{ name: 'semantic', evaluate: semanticAnalyzer },
|
|
434
|
+
]);
|
|
435
|
+
const result = await proceduralJudge.evaluate(evaluand);
|
|
436
|
+
|
|
437
|
+
// Reactive specialist-based evaluation
|
|
438
|
+
const reactiveJudge = new ReactiveJudge(router, specialists, deepDiveSpecialists);
|
|
439
|
+
const result = await reactiveJudge.evaluate(evaluand);
|
|
440
|
+
```
|
|
441
|
+
|
|
442
|
+
### Query Agent Evaluations
|
|
443
|
+
|
|
444
|
+
```javascript
|
|
445
|
+
// Filter by agent ID/name
|
|
446
|
+
obs_query_evaluations({
|
|
447
|
+
agentId: 'agent-123',
|
|
448
|
+
agentName: 'TaskRunner',
|
|
449
|
+
evaluationName: 'tool_correctness',
|
|
450
|
+
})
|
|
451
|
+
|
|
452
|
+
// Response includes agent-specific fields
|
|
453
|
+
{
|
|
454
|
+
stepScores: [{ step: 0, score: 0.9, explanation: '...' }],
|
|
455
|
+
toolVerifications: [{ toolName: 'search', toolCorrect: true, score: 1.0 }],
|
|
456
|
+
trajectoryLength: 5,
|
|
457
|
+
}
|
|
458
|
+
```
|
|
459
|
+
|
|
342
460
|
## Development
|
|
343
461
|
|
|
344
462
|
```bash
|
|
345
463
|
cd ~/.claude/mcp-servers/observability-toolkit
|
|
346
464
|
npm install
|
|
347
465
|
npm run build
|
|
348
|
-
npm test #
|
|
466
|
+
npm test # 3254 tests
|
|
349
467
|
npm run start
|
|
350
468
|
```
|
|
351
469
|
|
|
352
470
|
## Documentation
|
|
353
471
|
|
|
354
|
-
- [
|
|
355
|
-
- [docs/security.md](docs/security.md) - Security controls and hardening
|
|
356
|
-
- [docs/
|
|
357
|
-
- [docs/
|
|
472
|
+
- [docs/changelog/](docs/changelog/) - Version history and changelogs
|
|
473
|
+
- [docs/reliability/security.md](docs/reliability/security.md) - Security controls and hardening
|
|
474
|
+
- [docs/quality/llm-as-judge.md](docs/quality/llm-as-judge.md) - LLM-as-Judge architecture
|
|
475
|
+
- [docs/quality/agent-as-judge.md](docs/quality/agent-as-judge.md) - Agent-as-Judge architecture
|
|
476
|
+
- [docs/backlog/](docs/backlog/) - Feature backlog and roadmap
|
|
477
|
+
- [docs/changelog/SESSION_HISTORY.md](docs/changelog/SESSION_HISTORY.md) - Development session logs
|
|
478
|
+
- [docs/Summary.md](docs/Summary.md) - Full documentation index
|
package/dist/backends/index.d.ts
CHANGED
|
@@ -305,6 +305,142 @@ export declare const GENAI_EVALUATION_ATTRIBUTES: {
|
|
|
305
305
|
};
|
|
306
306
|
/** Evaluator types for LLM quality evaluation */
|
|
307
307
|
export type EvaluatorType = 'llm' | 'human' | 'rule' | 'classifier';
|
|
308
|
+
/**
|
|
309
|
+
* Custom OTel attribute extensions for Agent-as-Judge evaluation.
|
|
310
|
+
* These extend the official OTel GenAI semantic conventions (as of 2026-02).
|
|
311
|
+
* @see https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-events/
|
|
312
|
+
*/
|
|
313
|
+
export declare const AGENT_JUDGE_ATTRIBUTES: {
|
|
314
|
+
/** Custom: JSON array of per-step scores */
|
|
315
|
+
readonly STEP_SCORES: "gen_ai.evaluation.step_scores";
|
|
316
|
+
/** Custom: JSON array of tool verifications */
|
|
317
|
+
readonly TOOL_VERIFICATIONS: "gen_ai.evaluation.tool_verifications";
|
|
318
|
+
/** Custom: integer count of steps in agent trajectory */
|
|
319
|
+
readonly TRAJECTORY_LENGTH: "gen_ai.evaluation.trajectory_length";
|
|
320
|
+
};
|
|
321
|
+
/** Maximum recommended step scores per evaluation for performance */
|
|
322
|
+
export declare const MAX_STEP_SCORES = 1000;
|
|
323
|
+
/** Maximum recommended tool verifications per evaluation for performance */
|
|
324
|
+
export declare const MAX_TOOL_VERIFICATIONS = 500;
|
|
325
|
+
/** Maximum step identifier length */
|
|
326
|
+
export declare const MAX_STEP_ID_LENGTH = 256;
|
|
327
|
+
/**
|
|
328
|
+
* Per-step evaluation score for agent trajectory analysis.
|
|
329
|
+
* Captures evaluation of individual steps in an agent's execution.
|
|
330
|
+
*
|
|
331
|
+
* @remarks
|
|
332
|
+
* - Maximum 1000 steps recommended per evaluation for query performance
|
|
333
|
+
* - Step identifiers limited to 256 characters
|
|
334
|
+
* - Scores must be in normalized range [0, 1]
|
|
335
|
+
* - Evidence MUST be JSON-serializable (no functions, symbols, circular refs)
|
|
336
|
+
*
|
|
337
|
+
* Stored as custom OTel attribute: `gen_ai.evaluation.step_scores` (JSON array)
|
|
338
|
+
*
|
|
339
|
+
* @example
|
|
340
|
+
* ```typescript
|
|
341
|
+
* const stepScores: StepScore[] = [
|
|
342
|
+
* { step: 0, score: 0.9, explanation: 'Correct tool selection' },
|
|
343
|
+
* { step: 1, score: 0.7, explanation: 'Valid arguments but suboptimal' },
|
|
344
|
+
* { step: 'final_response', score: 1.0, evidence: { matched: true } },
|
|
345
|
+
* ];
|
|
346
|
+
* ```
|
|
347
|
+
*/
|
|
348
|
+
export interface StepScore {
|
|
349
|
+
/** Step identifier (max 256 chars) or zero-based index (non-negative integer) */
|
|
350
|
+
step: string | number;
|
|
351
|
+
/** Normalized score for this step (0-1 inclusive) */
|
|
352
|
+
score: number;
|
|
353
|
+
/**
|
|
354
|
+
* Evidence supporting the score.
|
|
355
|
+
* @security MUST be JSON-serializable (no functions, symbols, circular refs)
|
|
356
|
+
*/
|
|
357
|
+
evidence?: Record<string, unknown> | string | number | boolean | null;
|
|
358
|
+
/** Optional explanation for the score */
|
|
359
|
+
explanation?: string;
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Type guard for indexed step scores (numeric trajectory index).
|
|
363
|
+
* Use for type narrowing when processing agent trajectory steps.
|
|
364
|
+
*
|
|
365
|
+
* @param score - StepScore to check
|
|
366
|
+
* @returns true if step is a numeric index
|
|
367
|
+
*
|
|
368
|
+
* @example
|
|
369
|
+
* ```typescript
|
|
370
|
+
* if (isIndexedStep(score)) {
|
|
371
|
+
* console.log(`Step ${score.step + 1} of ${total}`); // step is number
|
|
372
|
+
* }
|
|
373
|
+
* ```
|
|
374
|
+
*/
|
|
375
|
+
export declare function isIndexedStep(score: StepScore): score is StepScore & {
|
|
376
|
+
step: number;
|
|
377
|
+
};
|
|
378
|
+
/**
|
|
379
|
+
* Type guard for named step scores (semantic step identifier).
|
|
380
|
+
* Use for type narrowing when processing named evaluation steps.
|
|
381
|
+
*
|
|
382
|
+
* @param score - StepScore to check
|
|
383
|
+
* @returns true if step is a string identifier
|
|
384
|
+
*
|
|
385
|
+
* @example
|
|
386
|
+
* ```typescript
|
|
387
|
+
* if (isNamedStep(score)) {
|
|
388
|
+
* console.log(`Evaluating: ${score.step}`); // step is string
|
|
389
|
+
* }
|
|
390
|
+
* ```
|
|
391
|
+
*/
|
|
392
|
+
export declare function isNamedStep(score: StepScore): score is StepScore & {
|
|
393
|
+
step: string;
|
|
394
|
+
};
|
|
395
|
+
/**
|
|
396
|
+
* Tool call verification result for agent evaluation.
|
|
397
|
+
* Records correctness of individual tool invocations.
|
|
398
|
+
*
|
|
399
|
+
* @remarks
|
|
400
|
+
* - Maximum 500 verifications recommended per evaluation for query performance
|
|
401
|
+
* - Score is typically a weighted average of toolCorrect, argsCorrect, resultCorrect
|
|
402
|
+
* - Score formula: (toolCorrect * 0.4 + argsCorrect * 0.3 + resultCorrect * 0.3)
|
|
403
|
+
*
|
|
404
|
+
* Stored as custom OTel attribute: `gen_ai.evaluation.tool_verifications` (JSON array)
|
|
405
|
+
*
|
|
406
|
+
* @example
|
|
407
|
+
* ```typescript
|
|
408
|
+
* const verification: ToolVerification = {
|
|
409
|
+
* toolName: 'search',
|
|
410
|
+
* toolCallId: 'call_abc123',
|
|
411
|
+
* toolCorrect: true,
|
|
412
|
+
* argsCorrect: true,
|
|
413
|
+
* resultCorrect: true,
|
|
414
|
+
* score: 1.0,
|
|
415
|
+
* expectedTool: 'search',
|
|
416
|
+
* evidence: { query: 'test', results: 5 },
|
|
417
|
+
* };
|
|
418
|
+
* ```
|
|
419
|
+
*/
|
|
420
|
+
export interface ToolVerification {
|
|
421
|
+
/** Name of the tool that was called */
|
|
422
|
+
toolName: string;
|
|
423
|
+
/** Tool call ID for correlation (gen_ai.tool.call.id) */
|
|
424
|
+
toolCallId?: string;
|
|
425
|
+
/** Whether the tool selection was correct */
|
|
426
|
+
toolCorrect: boolean;
|
|
427
|
+
/** Whether the arguments were valid/correct */
|
|
428
|
+
argsCorrect: boolean;
|
|
429
|
+
/** Whether the result was as expected (if verifiable) */
|
|
430
|
+
resultCorrect?: boolean;
|
|
431
|
+
/**
|
|
432
|
+
* Combined correctness score (0-1).
|
|
433
|
+
* Typically computed as weighted average of correctness flags.
|
|
434
|
+
*/
|
|
435
|
+
score: number;
|
|
436
|
+
/** Expected tool name (for comparison in evaluation) */
|
|
437
|
+
expectedTool?: string;
|
|
438
|
+
/**
|
|
439
|
+
* Evidence or execution result.
|
|
440
|
+
* @security MUST be JSON-serializable (no functions, symbols, circular refs)
|
|
441
|
+
*/
|
|
442
|
+
evidence?: Record<string, unknown> | string | number | boolean | null;
|
|
443
|
+
}
|
|
308
444
|
/**
|
|
309
445
|
* Normalized evaluation result from gen_ai.evaluation.result events
|
|
310
446
|
*/
|
|
@@ -333,6 +469,27 @@ export interface EvaluationResult {
|
|
|
333
469
|
spanId?: string;
|
|
334
470
|
/** Session ID for session-scoped queries */
|
|
335
471
|
sessionId?: string;
|
|
472
|
+
/** Subject agent ID being evaluated (gen_ai.agent.id) */
|
|
473
|
+
agentId?: string;
|
|
474
|
+
/** Subject agent name being evaluated (gen_ai.agent.name) */
|
|
475
|
+
agentName?: string;
|
|
476
|
+
/**
|
|
477
|
+
* Per-step evaluation breakdown for agent trajectory.
|
|
478
|
+
* - `undefined`: No trajectory evaluation performed
|
|
479
|
+
* - `[]`: Zero-step execution (rare, usually indicates error)
|
|
480
|
+
* Max recommended: 1000 steps
|
|
481
|
+
*/
|
|
482
|
+
stepScores?: StepScore[];
|
|
483
|
+
/**
|
|
484
|
+
* Tool call correctness verifications.
|
|
485
|
+
* Max recommended: 500 verifications
|
|
486
|
+
*/
|
|
487
|
+
toolVerifications?: ToolVerification[];
|
|
488
|
+
/**
|
|
489
|
+
* Number of steps/actions in agent trajectory (efficiency metric).
|
|
490
|
+
* Stored as: gen_ai.evaluation.trajectory_length
|
|
491
|
+
*/
|
|
492
|
+
trajectoryLength?: number;
|
|
336
493
|
}
|
|
337
494
|
/** Aggregation functions for evaluation scores */
|
|
338
495
|
export type EvaluationAggregation = 'avg' | 'min' | 'max' | 'count' | 'p50' | 'p95' | 'p99';
|
|
@@ -361,8 +518,14 @@ export interface EvaluationQueryOptions extends QueryOptions {
|
|
|
361
518
|
aggregation?: EvaluationAggregation;
|
|
362
519
|
/** Group aggregations by these fields */
|
|
363
520
|
groupBy?: EvaluationGroupByField[];
|
|
521
|
+
/** Filter by subject agent ID (gen_ai.agent.id) */
|
|
522
|
+
agentId?: string;
|
|
523
|
+
/** Filter by subject agent name (gen_ai.agent.name) */
|
|
524
|
+
agentName?: string;
|
|
364
525
|
}
|
|
365
526
|
export type { OTLPTraceExport, OTLPLogExport, OTLPMetricExport, } from '../lib/otlp-export.js';
|
|
527
|
+
export type { VerificationType, HumanVerificationEvent, VerificationQueryOptions, } from '../lib/verification-events.js';
|
|
528
|
+
export { VALID_VERIFICATION_TYPES, recordVerification, queryVerifications, } from '../lib/verification-events.js';
|
|
366
529
|
export interface TelemetryBackend {
|
|
367
530
|
name: string;
|
|
368
531
|
queryTraces(options: TraceQueryOptions): Promise<TraceSpan[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/backends/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;;GAGG;AACH,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,qEAAqE;IACrE,UAAU,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,CAAC;IAC1F,iFAAiF;IACjF,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IACnB,iFAAiF;IACjF,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;CAC7C;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,gFAAgF;IAChF,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,gDAAgD;IAChD,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,aAAa,GAAG,OAAO,GAAG,YAAY,CAAC;AAE5E,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC,SAAS,CAAC,EAAE,aAAa,CAAC;IAE1B,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAEvB,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;CACjD;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,CAAC;IAChC,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,CAAC;IAC7C,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,eAAO,MAAM,sBAAsB;;;;CAIzB,CAAC;AAEX,eAAO,MAAM,qBAAqB;;;;CAIxB,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,wBAAwB;;;;CAI3B,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,yBAAyB;;;CAG5B,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,sBAAsB;;;CAGzB,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,qBAAqB;;;;;CAKxB,CAAC;AAEX;;;;;;GAMG;AACH,eAAO,MAAM,eAAe;IAC1B,8BAA8B;;IAE9B,wBAAwB;;IAExB,gCAAgC;;IAEhC,2BAA2B;;IAE3B,gCAAgC;;IAEhC,uBAAuB;;IAEvB,oBAAoB;;IAEpB,wBAAwB;;IAExB,4BAA4B;;IAE5B,iCAAiC;;IAEjC,gBAAgB;;IAEhB,kBAAkB;;IAElB,WAAW;;IAEX,mBAAmB;;IAEnB,iBAAiB;;CAET,CAAC;AAEX,8CAA8C;AAC9C,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,OAAO,eAAe,CAAC,CAAC;AAEnF,+EAA+E;AAC/E,eAAO,MAAM,mBAAmB,EAAE,aAAa,EAAmC,CAAC;AAEnF,MAAM,WAAW,iBAAkB,SAAQ,YAAY;IACrD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC;IAC5D,yDAAyD;IACzD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,sDAAsD;IACtD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,iDAAiD;IACjD,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,gDAAgD;IAChD,aAAa,CAAC,EAAE,sBAAsB,EAAE,CAAC;IAEzC,2CAA2C;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mDAAmD;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAED,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,qDAAqD;IACrD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gDAAgD;IAChD,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,gDAAgD;IAChD,aAAa,CAAC,EAAE,sBAAsB,EAAE,CAAC;IACzC,+CAA+C;IAC/C,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,0GAA0G;IAC1G,cAAc,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;IAC9B,gFAAgF;IAChF,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACtD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;IACvF,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,mEAAmE;IACnE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,MAAM,GAAG,YAAY,GAAG,cAAc,GAAG,cAAc,CAAC;AAEzF,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yFAAyF;IACzF,aAAa,CAAC,EAAE,kBAAkB,CAAC;IACnC,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,eAAO,MAAM,2BAA2B;;;;;IAKtC,2EAA2E;;IAE3E,yEAAyE;;IAEzE,0CAA0C;;CAElC,CAAC;AAEX,iDAAiD;AACjD,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/backends/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;;GAGG;AACH,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,qEAAqE;IACrE,UAAU,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,CAAC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,CAAC;IAC1F,iFAAiF;IACjF,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IACnB,iFAAiF;IACjF,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;CAC7C;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,gFAAgF;IAChF,oBAAoB,CAAC,EAAE,oBAAoB,CAAC;IAC5C,gDAAgD;IAChD,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,sBAAsB,GAAG,aAAa,GAAG,OAAO,GAAG,YAAY,CAAC;AAE5E,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC,SAAS,CAAC,EAAE,aAAa,CAAC;IAE1B,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAEvB,sBAAsB,CAAC,EAAE,sBAAsB,CAAC;CACjD;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,CAAC;IAChC,IAAI,EAAE,CAAC,EAAE,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,CAAC;IAC7C,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,eAAO,MAAM,sBAAsB;;;;CAIzB,CAAC;AAEX,eAAO,MAAM,qBAAqB;;;;CAIxB,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,wBAAwB;;;;CAI3B,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,yBAAyB;;;CAG5B,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,sBAAsB;;;CAGzB,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,qBAAqB;;;;;CAKxB,CAAC;AAEX;;;;;;GAMG;AACH,eAAO,MAAM,eAAe;IAC1B,8BAA8B;;IAE9B,wBAAwB;;IAExB,gCAAgC;;IAEhC,2BAA2B;;IAE3B,gCAAgC;;IAEhC,uBAAuB;;IAEvB,oBAAoB;;IAEpB,wBAAwB;;IAExB,4BAA4B;;IAE5B,iCAAiC;;IAEjC,gBAAgB;;IAEhB,kBAAkB;;IAElB,WAAW;;IAEX,mBAAmB;;IAEnB,iBAAiB;;CAET,CAAC;AAEX,8CAA8C;AAC9C,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,OAAO,eAAe,CAAC,CAAC;AAEnF,+EAA+E;AAC/E,eAAO,MAAM,mBAAmB,EAAE,aAAa,EAAmC,CAAC;AAEnF,MAAM,WAAW,iBAAkB,SAAQ,YAAY;IACrD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC;IAC5D,yDAAyD;IACzD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,sDAAsD;IACtD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,iDAAiD;IACjD,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,gDAAgD;IAChD,aAAa,CAAC,EAAE,sBAAsB,EAAE,CAAC;IAEzC,2CAA2C;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mDAAmD;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAED,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,qDAAqD;IACrD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,gDAAgD;IAChD,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B,gDAAgD;IAChD,aAAa,CAAC,EAAE,sBAAsB,EAAE,CAAC;IACzC,+CAA+C;IAC/C,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,0GAA0G;IAC1G,cAAc,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;IAC9B,gFAAgF;IAChF,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACtD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;IACvF,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,mEAAmE;IACnE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,MAAM,GAAG,YAAY,GAAG,cAAc,GAAG,cAAc,CAAC;AAEzF,MAAM,WAAW,oBAAqB,SAAQ,YAAY;IACxD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,yFAAyF;IACzF,aAAa,CAAC,EAAE,kBAAkB,CAAC;IACnC,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;GAGG;AACH,eAAO,MAAM,2BAA2B;;;;;IAKtC,2EAA2E;;IAE3E,yEAAyE;;IAEzE,0CAA0C;;CAElC,CAAC;AAEX,iDAAiD;AACjD,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,YAAY,CAAC;AAMpE;;;;GAIG;AACH,eAAO,MAAM,sBAAsB;IACjC,4CAA4C;;IAE5C,+CAA+C;;IAE/C,yDAAyD;;CAEjD,CAAC;AAEX,qEAAqE;AACrE,eAAO,MAAM,eAAe,OAAO,CAAC;AAEpC,4EAA4E;AAC5E,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C,qCAAqC;AACrC,eAAO,MAAM,kBAAkB,MAAM,CAAC;AAEtC;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,SAAS;IACxB,iFAAiF;IACjF,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IACtB,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;IACtE,yCAAyC;IACzC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,GAAG,KAAK,IAAI,SAAS,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAErF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,KAAK,IAAI,SAAS,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAEnF;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,yDAAyD;IACzD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6CAA6C;IAC7C,WAAW,EAAE,OAAO,CAAC;IACrB,+CAA+C;IAC/C,WAAW,EAAE,OAAO,CAAC;IACrB,yDAAyD;IACzD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;CACvE;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,sCAAsC;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,cAAc,EAAE,MAAM,CAAC;IACvB,oDAAoD;IACpD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uDAAuD;IACvD,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,uDAAuD;IACvD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,yDAAyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IACzB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACvC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,kDAAkD;AAClD,MAAM,MAAM,qBAAqB,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;AAE5F,gEAAgE;AAChE,MAAM,MAAM,sBAAsB,GAAG,gBAAgB,GAAG,YAAY,GAAG,WAAW,CAAC;AAEnF,MAAM,WAAW,sBAAuB,SAAQ,YAAY;IAC1D,gEAAgE;IAChE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,+DAA+D;IAC/D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,4CAA4C;IAC5C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yBAAyB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sCAAsC;IACtC,WAAW,CAAC,EAAE,qBAAqB,CAAC;IACpC,yCAAyC;IACzC,OAAO,CAAC,EAAE,sBAAsB,EAAE,CAAC;IAGnC,mDAAmD;IACnD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAGD,YAAY,EACV,eAAe,EACf,aAAa,EACb,gBAAgB,GACjB,MAAM,uBAAuB,CAAC;AAG/B,YAAY,EACV,gBAAgB,EAChB,sBAAsB,EACtB,wBAAwB,GACzB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EACL,wBAAwB,EACxB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AAEvC,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IAEb,WAAW,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC9D,SAAS,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC1D,YAAY,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IACtE,cAAc,CAAC,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACpE,gBAAgB,CAAC,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAChF,WAAW,IAAI,OAAO,CAAC;QAAE,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAGrE,aAAa,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,iBAAiB,EAAE,UAAU,CAAC,CAAC;IAGvE,gBAAgB,CAAC,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,uBAAuB,EAAE,eAAe,CAAC,CAAC;IACxG,cAAc,CAAC,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,OAAO,uBAAuB,EAAE,aAAa,CAAC,CAAC;IAClG,iBAAiB,CAAC,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,OAAO,uBAAuB,EAAE,gBAAgB,CAAC,CAAC;CAC5G"}
|
package/dist/backends/index.js
CHANGED
|
@@ -107,4 +107,61 @@ export const GENAI_EVALUATION_ATTRIBUTES = {
|
|
|
107
107
|
/** Custom extension: Type of evaluator */
|
|
108
108
|
EVALUATOR_TYPE: 'gen_ai.evaluation.evaluator.type',
|
|
109
109
|
};
|
|
110
|
+
// ============================================================================
|
|
111
|
+
// Agent-as-Judge Types (Section 10.7)
|
|
112
|
+
// ============================================================================
|
|
113
|
+
/**
|
|
114
|
+
* Custom OTel attribute extensions for Agent-as-Judge evaluation.
|
|
115
|
+
* These extend the official OTel GenAI semantic conventions (as of 2026-02).
|
|
116
|
+
* @see https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-events/
|
|
117
|
+
*/
|
|
118
|
+
export const AGENT_JUDGE_ATTRIBUTES = {
|
|
119
|
+
/** Custom: JSON array of per-step scores */
|
|
120
|
+
STEP_SCORES: 'gen_ai.evaluation.step_scores',
|
|
121
|
+
/** Custom: JSON array of tool verifications */
|
|
122
|
+
TOOL_VERIFICATIONS: 'gen_ai.evaluation.tool_verifications',
|
|
123
|
+
/** Custom: integer count of steps in agent trajectory */
|
|
124
|
+
TRAJECTORY_LENGTH: 'gen_ai.evaluation.trajectory_length',
|
|
125
|
+
};
|
|
126
|
+
/** Maximum recommended step scores per evaluation for performance */
|
|
127
|
+
export const MAX_STEP_SCORES = 1000;
|
|
128
|
+
/** Maximum recommended tool verifications per evaluation for performance */
|
|
129
|
+
export const MAX_TOOL_VERIFICATIONS = 500;
|
|
130
|
+
/** Maximum step identifier length */
|
|
131
|
+
export const MAX_STEP_ID_LENGTH = 256;
|
|
132
|
+
/**
|
|
133
|
+
* Type guard for indexed step scores (numeric trajectory index).
|
|
134
|
+
* Use for type narrowing when processing agent trajectory steps.
|
|
135
|
+
*
|
|
136
|
+
* @param score - StepScore to check
|
|
137
|
+
* @returns true if step is a numeric index
|
|
138
|
+
*
|
|
139
|
+
* @example
|
|
140
|
+
* ```typescript
|
|
141
|
+
* if (isIndexedStep(score)) {
|
|
142
|
+
* console.log(`Step ${score.step + 1} of ${total}`); // step is number
|
|
143
|
+
* }
|
|
144
|
+
* ```
|
|
145
|
+
*/
|
|
146
|
+
export function isIndexedStep(score) {
|
|
147
|
+
return typeof score.step === 'number';
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Type guard for named step scores (semantic step identifier).
|
|
151
|
+
* Use for type narrowing when processing named evaluation steps.
|
|
152
|
+
*
|
|
153
|
+
* @param score - StepScore to check
|
|
154
|
+
* @returns true if step is a string identifier
|
|
155
|
+
*
|
|
156
|
+
* @example
|
|
157
|
+
* ```typescript
|
|
158
|
+
* if (isNamedStep(score)) {
|
|
159
|
+
* console.log(`Evaluating: ${score.step}`); // step is string
|
|
160
|
+
* }
|
|
161
|
+
* ```
|
|
162
|
+
*/
|
|
163
|
+
export function isNamedStep(score) {
|
|
164
|
+
return typeof score.step === 'string';
|
|
165
|
+
}
|
|
166
|
+
export { VALID_VERIFICATION_TYPES, recordVerification, queryVerifications, } from '../lib/verification-events.js';
|
|
110
167
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/backends/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAgIH;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,QAAQ,EAAE,iBAAiB;IAC3B,UAAU,EAAE,mBAAmB;IAC/B,iBAAiB,EAAE,0BAA0B;CACrC,CAAC;AAEX,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,SAAS,EAAE,kBAAkB;IAC7B,YAAY,EAAE,qBAAqB;IACnC,SAAS,EAAE,kBAAkB;CACrB,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,KAAK,EAAE,sBAAsB;IAC7B,WAAW,EAAE,4BAA4B;IACzC,UAAU,EAAE,2BAA2B;CAC/B,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,KAAK,EAAE,uBAAuB;IAC9B,cAAc,EAAE,gCAAgC;CACxC,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,YAAY,EAAE,2BAA2B;IACzC,aAAa,EAAE,4BAA4B;CACnC,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,cAAc,EAAE,uBAAuB;IACvC,aAAa,EAAE,sBAAsB;IACrC,MAAM,EAAE,eAAe;IACvB,eAAe,EAAE,wBAAwB;CACjC,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,8BAA8B;IAC9B,SAAS,EAAE,WAAW;IACtB,wBAAwB;IACxB,MAAM,EAAE,QAAQ;IAChB,gCAAgC;IAChC,WAAW,EAAE,aAAa;IAC1B,2BAA2B;IAC3B,YAAY,EAAE,iBAAiB;IAC/B,gCAAgC;IAChC,UAAU,EAAE,YAAY;IACxB,uBAAuB;IACvB,aAAa,EAAE,eAAe;IAC9B,oBAAoB;IACpB,MAAM,EAAE,QAAQ;IAChB,wBAAwB;IACxB,UAAU,EAAE,YAAY;IACxB,4BAA4B;IAC5B,MAAM,EAAE,QAAQ;IAChB,iCAAiC;IACjC,WAAW,EAAE,aAAa;IAC1B,gBAAgB;IAChB,SAAS,EAAE,WAAW;IACtB,kBAAkB;IAClB,WAAW,EAAE,aAAa;IAC1B,WAAW;IACX,IAAI,EAAE,MAAM;IACZ,mBAAmB;IACnB,YAAY,EAAE,cAAc;IAC5B,iBAAiB;IACjB,UAAU,EAAE,YAAY;CAChB,CAAC;AAKX,+EAA+E;AAC/E,MAAM,CAAC,MAAM,mBAAmB,GAAoB,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;AAqFnF;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG;IACzC,IAAI,EAAE,wBAAwB;IAC9B,WAAW,EAAE,+BAA+B;IAC5C,WAAW,EAAE,+BAA+B;IAC5C,WAAW,EAAE,+BAA+B;IAC5C,2EAA2E;IAC3E,UAAU,EAAE,8BAA8B;IAC1C,yEAAyE;IACzE,SAAS,EAAE,6BAA6B;IACxC,0CAA0C;IAC1C,cAAc,EAAE,kCAAkC;CAC1C,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/backends/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAgIH;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,QAAQ,EAAE,iBAAiB;IAC3B,UAAU,EAAE,mBAAmB;IAC/B,iBAAiB,EAAE,0BAA0B;CACrC,CAAC;AAEX,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,SAAS,EAAE,kBAAkB;IAC7B,YAAY,EAAE,qBAAqB;IACnC,SAAS,EAAE,kBAAkB;CACrB,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,KAAK,EAAE,sBAAsB;IAC7B,WAAW,EAAE,4BAA4B;IACzC,UAAU,EAAE,2BAA2B;CAC/B,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,KAAK,EAAE,uBAAuB;IAC9B,cAAc,EAAE,gCAAgC;CACxC,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,YAAY,EAAE,2BAA2B;IACzC,aAAa,EAAE,4BAA4B;CACnC,CAAC;AAEX;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,cAAc,EAAE,uBAAuB;IACvC,aAAa,EAAE,sBAAsB;IACrC,MAAM,EAAE,eAAe;IACvB,eAAe,EAAE,wBAAwB;CACjC,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,8BAA8B;IAC9B,SAAS,EAAE,WAAW;IACtB,wBAAwB;IACxB,MAAM,EAAE,QAAQ;IAChB,gCAAgC;IAChC,WAAW,EAAE,aAAa;IAC1B,2BAA2B;IAC3B,YAAY,EAAE,iBAAiB;IAC/B,gCAAgC;IAChC,UAAU,EAAE,YAAY;IACxB,uBAAuB;IACvB,aAAa,EAAE,eAAe;IAC9B,oBAAoB;IACpB,MAAM,EAAE,QAAQ;IAChB,wBAAwB;IACxB,UAAU,EAAE,YAAY;IACxB,4BAA4B;IAC5B,MAAM,EAAE,QAAQ;IAChB,iCAAiC;IACjC,WAAW,EAAE,aAAa;IAC1B,gBAAgB;IAChB,SAAS,EAAE,WAAW;IACtB,kBAAkB;IAClB,WAAW,EAAE,aAAa;IAC1B,WAAW;IACX,IAAI,EAAE,MAAM;IACZ,mBAAmB;IACnB,YAAY,EAAE,cAAc;IAC5B,iBAAiB;IACjB,UAAU,EAAE,YAAY;CAChB,CAAC;AAKX,+EAA+E;AAC/E,MAAM,CAAC,MAAM,mBAAmB,GAAoB,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;AAqFnF;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG;IACzC,IAAI,EAAE,wBAAwB;IAC9B,WAAW,EAAE,+BAA+B;IAC5C,WAAW,EAAE,+BAA+B;IAC5C,WAAW,EAAE,+BAA+B;IAC5C,2EAA2E;IAC3E,UAAU,EAAE,8BAA8B;IAC1C,yEAAyE;IACzE,SAAS,EAAE,6BAA6B;IACxC,0CAA0C;IAC1C,cAAc,EAAE,kCAAkC;CAC1C,CAAC;AAKX,+EAA+E;AAC/E,sCAAsC;AACtC,+EAA+E;AAE/E;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,4CAA4C;IAC5C,WAAW,EAAE,+BAA+B;IAC5C,+CAA+C;IAC/C,kBAAkB,EAAE,sCAAsC;IAC1D,yDAAyD;IACzD,iBAAiB,EAAE,qCAAqC;CAChD,CAAC;AAEX,qEAAqE;AACrE,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AAEpC,4EAA4E;AAC5E,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C,qCAAqC;AACrC,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAqCtC;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,aAAa,CAAC,KAAgB;IAC5C,OAAO,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;AACxC,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,WAAW,CAAC,KAAgB;IAC1C,OAAO,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;AACxC,CAAC;AA4JD,OAAO,EACL,wBAAwB,EACxB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,+BAA+B,CAAC"}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { describe, it } from 'node:test';
|
|
5
5
|
import assert from 'node:assert';
|
|
6
|
-
import { GENAI_PROVIDERS, SUPPORTED_PROVIDERS, GENAI_CORE_ATTRIBUTES, GENAI_AGENT_ATTRIBUTES, GENAI_TOOL_ATTRIBUTES, GENAI_REQUEST_ATTRIBUTES, GENAI_RESPONSE_ATTRIBUTES, GENAI_USAGE_ATTRIBUTES, GENAI_EVALUATION_ATTRIBUTES, } from './index.js';
|
|
6
|
+
import { GENAI_PROVIDERS, SUPPORTED_PROVIDERS, GENAI_CORE_ATTRIBUTES, GENAI_AGENT_ATTRIBUTES, GENAI_TOOL_ATTRIBUTES, GENAI_REQUEST_ATTRIBUTES, GENAI_RESPONSE_ATTRIBUTES, GENAI_USAGE_ATTRIBUTES, GENAI_EVALUATION_ATTRIBUTES, isIndexedStep, isNamedStep, } from './index.js';
|
|
7
7
|
describe('OTel GenAI Provider Constants', () => {
|
|
8
8
|
describe('GENAI_PROVIDERS', () => {
|
|
9
9
|
it('should include Anthropic provider', () => {
|
|
@@ -153,4 +153,58 @@ describe('GenAIOperationName type', () => {
|
|
|
153
153
|
assert.strictEqual(ops.length, 4);
|
|
154
154
|
});
|
|
155
155
|
});
|
|
156
|
+
describe('StepScore type guards', () => {
|
|
157
|
+
describe('isIndexedStep', () => {
|
|
158
|
+
it('should return true for numeric step', () => {
|
|
159
|
+
const score = { step: 0, score: 0.9 };
|
|
160
|
+
assert.strictEqual(isIndexedStep(score), true);
|
|
161
|
+
});
|
|
162
|
+
it('should return true for positive numeric step', () => {
|
|
163
|
+
const score = { step: 5, score: 0.7 };
|
|
164
|
+
assert.strictEqual(isIndexedStep(score), true);
|
|
165
|
+
});
|
|
166
|
+
it('should return false for string step', () => {
|
|
167
|
+
const score = { step: 'final_response', score: 1.0 };
|
|
168
|
+
assert.strictEqual(isIndexedStep(score), false);
|
|
169
|
+
});
|
|
170
|
+
it('should enable type narrowing for number', () => {
|
|
171
|
+
const score = { step: 3, score: 0.8 };
|
|
172
|
+
if (isIndexedStep(score)) {
|
|
173
|
+
// TypeScript should narrow step to number
|
|
174
|
+
const nextStep = score.step + 1;
|
|
175
|
+
assert.strictEqual(nextStep, 4);
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
});
|
|
179
|
+
describe('isNamedStep', () => {
|
|
180
|
+
it('should return true for string step', () => {
|
|
181
|
+
const score = { step: 'reasoning', score: 0.85 };
|
|
182
|
+
assert.strictEqual(isNamedStep(score), true);
|
|
183
|
+
});
|
|
184
|
+
it('should return true for empty string step', () => {
|
|
185
|
+
const score = { step: '', score: 0.5 };
|
|
186
|
+
assert.strictEqual(isNamedStep(score), true);
|
|
187
|
+
});
|
|
188
|
+
it('should return false for numeric step', () => {
|
|
189
|
+
const score = { step: 0, score: 0.9 };
|
|
190
|
+
assert.strictEqual(isNamedStep(score), false);
|
|
191
|
+
});
|
|
192
|
+
it('should enable type narrowing for string', () => {
|
|
193
|
+
const score = { step: 'tool_selection', score: 0.95 };
|
|
194
|
+
if (isNamedStep(score)) {
|
|
195
|
+
// TypeScript should narrow step to string
|
|
196
|
+
const upper = score.step.toUpperCase();
|
|
197
|
+
assert.strictEqual(upper, 'TOOL_SELECTION');
|
|
198
|
+
}
|
|
199
|
+
});
|
|
200
|
+
});
|
|
201
|
+
describe('mutual exclusivity', () => {
|
|
202
|
+
it('should have exactly one guard return true', () => {
|
|
203
|
+
const indexedScore = { step: 0, score: 0.9 };
|
|
204
|
+
const namedScore = { step: 'final', score: 1.0 };
|
|
205
|
+
assert.strictEqual(isIndexedStep(indexedScore) !== isNamedStep(indexedScore), true);
|
|
206
|
+
assert.strictEqual(isIndexedStep(namedScore) !== isNamedStep(namedScore), true);
|
|
207
|
+
});
|
|
208
|
+
});
|
|
209
|
+
});
|
|
156
210
|
//# sourceMappingURL=index.test.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../../src/backends/index.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,qBAAqB,EACrB,sBAAsB,EACtB,qBAAqB,EACrB,wBAAwB,EACxB,yBAAyB,EACzB,sBAAsB,EACtB,2BAA2B,
|
|
1
|
+
{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../../src/backends/index.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EACL,eAAe,EACf,mBAAmB,EACnB,qBAAqB,EACrB,sBAAsB,EACtB,qBAAqB,EACrB,wBAAwB,EACxB,yBAAyB,EACzB,sBAAsB,EACtB,2BAA2B,EAC3B,aAAa,EACb,WAAW,GAIZ,MAAM,YAAY,CAAC;AAEpB,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;IAC7C,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;QACrE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,EAAE,CAAC,mBAAmB,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,CAAC,EAAE,CAAC,mBAAmB,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;YAC/B,MAAM,CAAC,EAAE,CAAC,mBAAmB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,CAAC,EAAE,CAAC,mBAAmB,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;YACtD,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;gBACtC,MAAM,CAAC,EAAE,CAAC,mBAAmB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,qBAAqB,QAAQ,EAAE,CAAC,CAAC;YACrF,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;YAChE,MAAM,CAAC,WAAW,CAAC,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC;QACtF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,SAAS,GAAoB;gBACjC,WAAW;gBACX,QAAQ;gBACR,aAAa;gBACb,iBAAiB;gBACjB,YAAY;gBACZ,eAAe;gBACf,QAAQ;gBACR,YAAY;aACb,CAAC;YACF,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,gCAAgC,EAAE,GAAG,EAAE;IAC9C,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,cAAc,EAAE,uBAAuB,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,aAAa,EAAE,sBAAsB,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;YAC5B,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,eAAe,EAAE,wBAAwB,CAAC,CAAC;QACtF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC9B,MAAM,CAAC,WAAW,CAAC,sBAAsB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;QACzE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;YAChC,MAAM,CAAC,WAAW,CAAC,sBAAsB,CAAC,UAAU,EAAE,mBAAmB,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;YAC/B,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;QAChF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;QACxC,EAAE,CAAC,mBAAmB,EAAE,GAAG,EAAE;YAC3B,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,KAAK,EAAE,sBAAsB,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,WAAW,EAAE,4BAA4B,CAAC,CAAC;QACzF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;YAChC,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,UAAU,EAAE,2BAA2B,CAAC,CAAC;QACvF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACzC,EAAE,CAAC,mBAAmB,EAAE,GAAG,EAAE;YAC3B,MAAM,CAAC,WAAW,CAAC,yBAAyB,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,CAAC,WAAW,CAAC,yBAAyB,CAAC,cAAc,EAAE,gCAAgC,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,CAAC,WAAW,CAAC,sBAAsB,CAAC,YAAY,EAAE,2BAA2B,CAAC,CAAC;QACvF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,CAAC,WAAW,CAAC,sBAAsB,CAAC,aAAa,EAAE,4BAA4B,CAAC,CAAC;QACzF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,6BAA6B,EAAE,GAAG,EAAE;QAC3C,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,CAAC,WAAW,CAAC,2BAA2B,CAAC,IAAI,EAAE,wBAAwB,CAAC,CAAC;QACjF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,CAAC,WAAW,CAAC,2BAA2B,CAAC,WAAW,EAAE,+BAA+B,CAAC,CAAC;QAC/F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,CAAC,WAAW,CAAC,2BAA2B,CAAC,WAAW,EAAE,+BAA+B,CAAC,CAAC;QAC/F,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,GAAG,GAAyB,CAAC,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,CAAC,CAAC;QACzF,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,KAAK,GAAc,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACjD,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,KAAK,GAAc,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACjD,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,KAAK,GAAc,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YAChE,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,KAAK,GAAc,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACjD,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,0CAA0C;gBAC1C,MAAM,QAAQ,GAAW,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;gBACxC,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,KAAK,GAAc,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YAC5D,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,KAAK,GAAc,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YAClD,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,KAAK,GAAc,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACjD,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,KAAK,GAAc,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YACjE,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,0CAA0C;gBAC1C,MAAM,KAAK,GAAW,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC/C,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,YAAY,GAAc,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACxD,MAAM,UAAU,GAAc,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YAE5D,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,YAAY,CAAC,KAAK,WAAW,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,CAAC;YACpF,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,UAAU,CAAC,KAAK,WAAW,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
* span or log record, not the batched OpenTelemetry export format.
|
|
6
6
|
*/
|
|
7
7
|
import { TelemetryBackend, TraceSpan, LogRecord, MetricDataPoint, LLMEvent, EvaluationResult, TraceQueryOptions, LogQueryOptions, MetricQueryOptions, LLMEventQueryOptions, EvaluationQueryOptions, OTLPTraceExport, OTLPLogExport, OTLPMetricExport } from './index.js';
|
|
8
|
+
import { type HistogramStats } from '../lib/histogram.js';
|
|
9
|
+
import { type AggregateParseStats } from '../lib/parse-stats.js';
|
|
8
10
|
export declare class LocalJsonlBackend implements TelemetryBackend {
|
|
9
11
|
name: string;
|
|
10
12
|
private telemetryDir;
|
|
@@ -15,6 +17,9 @@ export declare class LocalJsonlBackend implements TelemetryBackend {
|
|
|
15
17
|
private evaluationCache;
|
|
16
18
|
private useIndexes;
|
|
17
19
|
private circuitBreaker;
|
|
20
|
+
private parseStatsTracker;
|
|
21
|
+
/** Histograms for tracking query latency per type */
|
|
22
|
+
private queryHistograms;
|
|
18
23
|
constructor(telemetryDir?: string, useIndexes?: boolean);
|
|
19
24
|
/**
|
|
20
25
|
* Get circuit breaker state (for health check and testing)
|
|
@@ -32,6 +37,23 @@ export declare class LocalJsonlBackend implements TelemetryBackend {
|
|
|
32
37
|
* Get cache statistics for all query caches
|
|
33
38
|
*/
|
|
34
39
|
getCacheStats(): Record<string, import('../lib/cache.js').CacheStats>;
|
|
40
|
+
/**
|
|
41
|
+
* Get parse statistics for JSONL file processing.
|
|
42
|
+
* Returns aggregate stats including success rate and worst files.
|
|
43
|
+
*/
|
|
44
|
+
getParseStats(): AggregateParseStats;
|
|
45
|
+
/**
|
|
46
|
+
* Reset parse statistics (useful for testing)
|
|
47
|
+
*/
|
|
48
|
+
resetParseStats(): void;
|
|
49
|
+
/**
|
|
50
|
+
* Get query latency statistics per query type
|
|
51
|
+
*/
|
|
52
|
+
getQueryStats(): Record<string, HistogramStats>;
|
|
53
|
+
/**
|
|
54
|
+
* Reset query latency histograms (useful for testing)
|
|
55
|
+
*/
|
|
56
|
+
resetQueryStats(): void;
|
|
35
57
|
/**
|
|
36
58
|
* Try to use an index for a file, returning matching line numbers or null if full scan needed
|
|
37
59
|
*/
|
|
@@ -97,6 +119,14 @@ export declare class MultiDirectoryBackend implements TelemetryBackend {
|
|
|
97
119
|
* Get aggregated cache statistics from all backends
|
|
98
120
|
*/
|
|
99
121
|
getCacheStats(): Record<string, import('../lib/cache.js').CacheStats>;
|
|
122
|
+
/**
|
|
123
|
+
* Get aggregated parse statistics from all backends
|
|
124
|
+
*/
|
|
125
|
+
getParseStats(): AggregateParseStats;
|
|
126
|
+
/**
|
|
127
|
+
* Get aggregated query latency statistics from all backends
|
|
128
|
+
*/
|
|
129
|
+
getQueryStats(): Record<string, HistogramStats>;
|
|
100
130
|
/**
|
|
101
131
|
* Export traces in OTLP JSON format
|
|
102
132
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local-jsonl.d.ts","sourceRoot":"","sources":["../../src/backends/local-jsonl.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EACL,gBAAgB,EAChB,SAAS,EACT,SAAS,EACT,eAAe,EACf,QAAQ,EACR,gBAAgB,
|
|
1
|
+
{"version":3,"file":"local-jsonl.d.ts","sourceRoot":"","sources":["../../src/backends/local-jsonl.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EACL,gBAAgB,EAChB,SAAS,EACT,SAAS,EACT,eAAe,EACf,QAAQ,EACR,gBAAgB,EAIhB,iBAAiB,EACjB,eAAe,EACf,kBAAkB,EAClB,oBAAoB,EACpB,sBAAsB,EAOtB,eAAe,EACf,aAAa,EACb,gBAAgB,EAIjB,MAAM,YAAY,CAAC;AAiBpB,OAAO,EAAa,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAcrE,OAAO,EAAqB,KAAK,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAupBpF,qBAAa,iBAAkB,YAAW,gBAAgB;IACxD,IAAI,SAAiB;IACrB,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,UAAU,CAAqD;IACvE,OAAO,CAAC,QAAQ,CAAmD;IACnE,OAAO,CAAC,WAAW,CAA4D;IAC/E,OAAO,CAAC,aAAa,CAAuD;IAC5E,OAAO,CAAC,eAAe,CAAiE;IACxF,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,iBAAiB,CAAoB;IAE7C,qDAAqD;IACrD,OAAO,CAAC,eAAe,CAMrB;gBAEU,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,GAAE,OAAc;IAO7D;;OAEG;IACH,sBAAsB,IAAI,QAAQ,GAAG,MAAM,GAAG,WAAW;IAIzD;;OAEG;IACH,mBAAmB,IAAI,IAAI;IAI3B;;OAEG;IACH,UAAU,IAAI,IAAI;IAQlB;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,iBAAiB,EAAE,UAAU,CAAC;IAUrE;;;OAGG;IACH,aAAa,IAAI,mBAAmB;IAIpC;;OAEG;IACH,eAAe,IAAI,IAAI;IAIvB;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC;IAU/C;;OAEG;IACH,eAAe,IAAI,IAAI;IAQvB;;OAEG;IACH,OAAO,CAAC,WAAW;IAgBb,WAAW,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IA8K7D,SAAS,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAgKzD,YAAY,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAwH3E,OAAO,CAAC,SAAS;IA4FjB,OAAO,CAAC,mBAAmB;IAO3B;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAuBf,cAAc,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IA8JlE,gBAAgB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IA+J9E,WAAW,IAAI,OAAO,CAAC;QAAE,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAuC1E;;OAEG;IACG,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,eAAe,CAAC;IAK5E;;OAEG;IACG,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC;IAKtE;;OAEG;IACG,iBAAiB,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAIhF;AAED;;;GAGG;AACH,qBAAa,qBAAsB,YAAW,gBAAgB;IAC5D,IAAI,SAAqB;IACzB,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,WAAW,CAAsD;gBAE7D,GAAG,CAAC,EAAE,MAAM,EAAE,UAAU,GAAE,OAAc;IAKpD,cAAc,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAA;KAAE,CAAC;IAI/D,WAAW,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAwB7D,SAAS,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAwBzD,YAAY,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAarE,cAAc,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAwBlE,gBAAgB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAwB9E,WAAW,IAAI,OAAO,CAAC;QAAE,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAyBjJ;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,iBAAiB,EAAE,UAAU,CAAC;IAmCrE;;OAEG;IACH,aAAa,IAAI,mBAAmB;IA+CpC;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC;IAwD/C;;OAEG;IACG,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,eAAe,CAAC;IAK5E;;OAEG;IACG,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC;IAKtE;;OAEG;IACG,iBAAiB,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAIhF"}
|