observability-toolkit 1.8.4 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -5
- package/dist/backends/index.d.ts +163 -0
- package/dist/backends/index.d.ts.map +1 -1
- package/dist/backends/index.js +57 -0
- package/dist/backends/index.js.map +1 -1
- package/dist/backends/index.test.js +55 -1
- package/dist/backends/index.test.js.map +1 -1
- package/dist/backends/local-jsonl-boolean-search.test.js +8 -8
- package/dist/backends/local-jsonl-boolean-search.test.js.map +1 -1
- package/dist/backends/local-jsonl-cache.test.d.ts +2 -0
- package/dist/backends/local-jsonl-cache.test.d.ts.map +1 -0
- package/dist/backends/local-jsonl-cache.test.js +295 -0
- package/dist/backends/local-jsonl-cache.test.js.map +1 -0
- package/dist/backends/local-jsonl-circuit-breaker.test.d.ts +2 -0
- package/dist/backends/local-jsonl-circuit-breaker.test.d.ts.map +1 -0
- package/dist/backends/local-jsonl-circuit-breaker.test.js +180 -0
- package/dist/backends/local-jsonl-circuit-breaker.test.js.map +1 -0
- package/dist/backends/local-jsonl-export.test.d.ts +2 -0
- package/dist/backends/local-jsonl-export.test.d.ts.map +1 -0
- package/dist/backends/local-jsonl-export.test.js +704 -0
- package/dist/backends/local-jsonl-export.test.js.map +1 -0
- package/dist/backends/local-jsonl-index.test.d.ts +2 -0
- package/dist/backends/local-jsonl-index.test.d.ts.map +1 -0
- package/dist/backends/local-jsonl-index.test.js +554 -0
- package/dist/backends/local-jsonl-index.test.js.map +1 -0
- package/dist/backends/local-jsonl-logs.test.js +52 -43
- package/dist/backends/local-jsonl-logs.test.js.map +1 -1
- package/dist/backends/local-jsonl-metrics.test.d.ts +2 -0
- package/dist/backends/local-jsonl-metrics.test.d.ts.map +1 -0
- package/dist/backends/local-jsonl-metrics.test.js +876 -0
- package/dist/backends/local-jsonl-metrics.test.js.map +1 -0
- package/dist/backends/local-jsonl-traces.test.js +89 -83
- package/dist/backends/local-jsonl-traces.test.js.map +1 -1
- package/dist/backends/local-jsonl.d.ts +39 -0
- package/dist/backends/local-jsonl.d.ts.map +1 -1
- package/dist/backends/local-jsonl.js +975 -492
- package/dist/backends/local-jsonl.js.map +1 -1
- package/dist/backends/signoz-api-circuit-breaker.test.d.ts +6 -0
- package/dist/backends/signoz-api-circuit-breaker.test.d.ts.map +1 -0
- package/dist/backends/signoz-api-circuit-breaker.test.js +548 -0
- package/dist/backends/signoz-api-circuit-breaker.test.js.map +1 -0
- package/dist/backends/signoz-api-rate-limiter.test.d.ts +6 -0
- package/dist/backends/signoz-api-rate-limiter.test.d.ts.map +1 -0
- package/dist/backends/signoz-api-rate-limiter.test.js +390 -0
- package/dist/backends/signoz-api-rate-limiter.test.js.map +1 -0
- package/dist/backends/signoz-api-ssrf.test.d.ts +6 -0
- package/dist/backends/signoz-api-ssrf.test.d.ts.map +1 -0
- package/dist/backends/signoz-api-ssrf.test.js +216 -0
- package/dist/backends/signoz-api-ssrf.test.js.map +1 -0
- package/dist/backends/signoz-api-test-helpers.d.ts +80 -0
- package/dist/backends/signoz-api-test-helpers.d.ts.map +1 -0
- package/dist/backends/signoz-api-test-helpers.js +79 -0
- package/dist/backends/signoz-api-test-helpers.js.map +1 -0
- package/dist/backends/signoz-api.d.ts +31 -1
- package/dist/backends/signoz-api.d.ts.map +1 -1
- package/dist/backends/signoz-api.js +717 -539
- package/dist/backends/signoz-api.js.map +1 -1
- package/dist/backends/signoz-api.test.d.ts +9 -0
- package/dist/backends/signoz-api.test.d.ts.map +1 -1
- package/dist/backends/signoz-api.test.js +20 -1032
- package/dist/backends/signoz-api.test.js.map +1 -1
- package/dist/lib/agent-as-judge.d.ts +388 -0
- package/dist/lib/agent-as-judge.d.ts.map +1 -0
- package/dist/lib/agent-as-judge.js +740 -0
- package/dist/lib/agent-as-judge.js.map +1 -0
- package/dist/lib/agent-as-judge.test.d.ts +5 -0
- package/dist/lib/agent-as-judge.test.d.ts.map +1 -0
- package/dist/lib/agent-as-judge.test.js +816 -0
- package/dist/lib/agent-as-judge.test.js.map +1 -0
- package/dist/lib/cache.d.ts +61 -2
- package/dist/lib/cache.d.ts.map +1 -1
- package/dist/lib/cache.js +54 -3
- package/dist/lib/cache.js.map +1 -1
- package/dist/lib/circuit-breaker.d.ts +101 -0
- package/dist/lib/circuit-breaker.d.ts.map +1 -0
- package/dist/lib/circuit-breaker.js +158 -0
- package/dist/lib/circuit-breaker.js.map +1 -0
- package/dist/lib/circuit-breaker.test.d.ts +2 -0
- package/dist/lib/circuit-breaker.test.d.ts.map +1 -0
- package/dist/lib/circuit-breaker.test.js +263 -0
- package/dist/lib/circuit-breaker.test.js.map +1 -0
- package/dist/lib/confident-export.d.ts +101 -0
- package/dist/lib/confident-export.d.ts.map +1 -0
- package/dist/lib/confident-export.js +393 -0
- package/dist/lib/confident-export.js.map +1 -0
- package/dist/lib/confident-export.test.d.ts +7 -0
- package/dist/lib/confident-export.test.d.ts.map +1 -0
- package/dist/lib/confident-export.test.js +835 -0
- package/dist/lib/confident-export.test.js.map +1 -0
- package/dist/lib/constants-symlink.test.d.ts +12 -0
- package/dist/lib/constants-symlink.test.d.ts.map +1 -0
- package/dist/lib/constants-symlink.test.js +357 -0
- package/dist/lib/constants-symlink.test.js.map +1 -0
- package/dist/lib/constants.d.ts +75 -0
- package/dist/lib/constants.d.ts.map +1 -1
- package/dist/lib/constants.js +104 -1
- package/dist/lib/constants.js.map +1 -1
- package/dist/lib/datadog-export.d.ts +156 -0
- package/dist/lib/datadog-export.d.ts.map +1 -0
- package/dist/lib/datadog-export.js +464 -0
- package/dist/lib/datadog-export.js.map +1 -0
- package/dist/lib/datadog-export.test.d.ts +14 -0
- package/dist/lib/datadog-export.test.d.ts.map +1 -0
- package/dist/lib/datadog-export.test.js +890 -0
- package/dist/lib/datadog-export.test.js.map +1 -0
- package/dist/lib/edge-cases.test.js +17 -17
- package/dist/lib/edge-cases.test.js.map +1 -1
- package/dist/lib/error-sanitizer.d.ts.map +1 -1
- package/dist/lib/error-sanitizer.js +29 -3
- package/dist/lib/error-sanitizer.js.map +1 -1
- package/dist/lib/error-sanitizer.test.js +159 -0
- package/dist/lib/error-sanitizer.test.js.map +1 -1
- package/dist/lib/error-types.d.ts +54 -0
- package/dist/lib/error-types.d.ts.map +1 -0
- package/dist/lib/error-types.js +154 -0
- package/dist/lib/error-types.js.map +1 -0
- package/dist/lib/error-types.test.d.ts +2 -0
- package/dist/lib/error-types.test.d.ts.map +1 -0
- package/dist/lib/error-types.test.js +196 -0
- package/dist/lib/error-types.test.js.map +1 -0
- package/dist/lib/evaluation-hooks.d.ts +49 -0
- package/dist/lib/evaluation-hooks.d.ts.map +1 -0
- package/dist/lib/evaluation-hooks.js +488 -0
- package/dist/lib/evaluation-hooks.js.map +1 -0
- package/dist/lib/evaluation-hooks.test.d.ts +8 -0
- package/dist/lib/evaluation-hooks.test.d.ts.map +1 -0
- package/dist/lib/evaluation-hooks.test.js +624 -0
- package/dist/lib/evaluation-hooks.test.js.map +1 -0
- package/dist/lib/export-utils.d.ts +99 -0
- package/dist/lib/export-utils.d.ts.map +1 -0
- package/dist/lib/export-utils.js +238 -0
- package/dist/lib/export-utils.js.map +1 -0
- package/dist/lib/export-utils.test.d.ts +5 -0
- package/dist/lib/export-utils.test.d.ts.map +1 -0
- package/dist/lib/export-utils.test.js +193 -0
- package/dist/lib/export-utils.test.js.map +1 -0
- package/dist/lib/file-utils.d.ts +17 -2
- package/dist/lib/file-utils.d.ts.map +1 -1
- package/dist/lib/file-utils.js +24 -5
- package/dist/lib/file-utils.js.map +1 -1
- package/dist/lib/file-utils.test.js +30 -0
- package/dist/lib/file-utils.test.js.map +1 -1
- package/dist/lib/histogram.d.ts +119 -0
- package/dist/lib/histogram.d.ts.map +1 -0
- package/dist/lib/histogram.js +202 -0
- package/dist/lib/histogram.js.map +1 -0
- package/dist/lib/histogram.test.d.ts +5 -0
- package/dist/lib/histogram.test.d.ts.map +1 -0
- package/dist/lib/histogram.test.js +381 -0
- package/dist/lib/histogram.test.js.map +1 -0
- package/dist/lib/indexer.test.js +27 -27
- package/dist/lib/indexer.test.js.map +1 -1
- package/dist/lib/input-validator.d.ts +12 -0
- package/dist/lib/input-validator.d.ts.map +1 -1
- package/dist/lib/input-validator.fuzz.test.d.ts +12 -0
- package/dist/lib/input-validator.fuzz.test.d.ts.map +1 -0
- package/dist/lib/input-validator.fuzz.test.js +290 -0
- package/dist/lib/input-validator.fuzz.test.js.map +1 -0
- package/dist/lib/input-validator.js +57 -3
- package/dist/lib/input-validator.js.map +1 -1
- package/dist/lib/input-validator.test.js +129 -1
- package/dist/lib/input-validator.test.js.map +1 -1
- package/dist/lib/instrumentation.d.ts +153 -0
- package/dist/lib/instrumentation.d.ts.map +1 -0
- package/dist/lib/instrumentation.integration.test.d.ts +2 -0
- package/dist/lib/instrumentation.integration.test.d.ts.map +1 -0
- package/dist/lib/instrumentation.integration.test.js +589 -0
- package/dist/lib/instrumentation.integration.test.js.map +1 -0
- package/dist/lib/instrumentation.js +520 -0
- package/dist/lib/instrumentation.js.map +1 -0
- package/dist/lib/instrumentation.test.d.ts +2 -0
- package/dist/lib/instrumentation.test.d.ts.map +1 -0
- package/dist/lib/instrumentation.test.js +821 -0
- package/dist/lib/instrumentation.test.js.map +1 -0
- package/dist/lib/langfuse-export.d.ts +125 -0
- package/dist/lib/langfuse-export.d.ts.map +1 -0
- package/dist/lib/langfuse-export.js +367 -0
- package/dist/lib/langfuse-export.js.map +1 -0
- package/dist/lib/langfuse-export.test.d.ts +7 -0
- package/dist/lib/langfuse-export.test.d.ts.map +1 -0
- package/dist/lib/langfuse-export.test.js +1007 -0
- package/dist/lib/langfuse-export.test.js.map +1 -0
- package/dist/lib/llm-as-judge.d.ts +657 -0
- package/dist/lib/llm-as-judge.d.ts.map +1 -0
- package/dist/lib/llm-as-judge.js +1397 -0
- package/dist/lib/llm-as-judge.js.map +1 -0
- package/dist/lib/llm-as-judge.test.d.ts +2 -0
- package/dist/lib/llm-as-judge.test.d.ts.map +1 -0
- package/dist/lib/llm-as-judge.test.js +2409 -0
- package/dist/lib/llm-as-judge.test.js.map +1 -0
- package/dist/lib/logger.d.ts +46 -0
- package/dist/lib/logger.d.ts.map +1 -0
- package/dist/lib/logger.js +81 -0
- package/dist/lib/logger.js.map +1 -0
- package/dist/lib/logger.test.d.ts +2 -0
- package/dist/lib/logger.test.d.ts.map +1 -0
- package/dist/lib/logger.test.js +122 -0
- package/dist/lib/logger.test.js.map +1 -0
- package/dist/lib/metrics.d.ts +62 -0
- package/dist/lib/metrics.d.ts.map +1 -0
- package/dist/lib/metrics.js +166 -0
- package/dist/lib/metrics.js.map +1 -0
- package/dist/lib/metrics.test.d.ts +5 -0
- package/dist/lib/metrics.test.d.ts.map +1 -0
- package/dist/lib/metrics.test.js +189 -0
- package/dist/lib/metrics.test.js.map +1 -0
- package/dist/lib/parse-stats.d.ts +119 -0
- package/dist/lib/parse-stats.d.ts.map +1 -0
- package/dist/lib/parse-stats.js +206 -0
- package/dist/lib/parse-stats.js.map +1 -0
- package/dist/lib/parse-stats.test.d.ts +5 -0
- package/dist/lib/parse-stats.test.d.ts.map +1 -0
- package/dist/lib/parse-stats.test.js +283 -0
- package/dist/lib/parse-stats.test.js.map +1 -0
- package/dist/lib/phoenix-export.d.ts +109 -0
- package/dist/lib/phoenix-export.d.ts.map +1 -0
- package/dist/lib/phoenix-export.js +429 -0
- package/dist/lib/phoenix-export.js.map +1 -0
- package/dist/lib/phoenix-export.test.d.ts +11 -0
- package/dist/lib/phoenix-export.test.d.ts.map +1 -0
- package/dist/lib/phoenix-export.test.js +725 -0
- package/dist/lib/phoenix-export.test.js.map +1 -0
- package/dist/lib/server-utils.d.ts +14 -1
- package/dist/lib/server-utils.d.ts.map +1 -1
- package/dist/lib/server-utils.js +43 -3
- package/dist/lib/server-utils.js.map +1 -1
- package/dist/lib/shared-schemas.d.ts +28 -0
- package/dist/lib/shared-schemas.d.ts.map +1 -1
- package/dist/lib/shared-schemas.js +33 -4
- package/dist/lib/shared-schemas.js.map +1 -1
- package/dist/lib/toon-encoder.d.ts +7 -2
- package/dist/lib/toon-encoder.d.ts.map +1 -1
- package/dist/lib/toon-encoder.js +21 -6
- package/dist/lib/toon-encoder.js.map +1 -1
- package/dist/lib/toon-encoder.test.d.ts +5 -0
- package/dist/lib/toon-encoder.test.d.ts.map +1 -0
- package/dist/lib/toon-encoder.test.js +85 -0
- package/dist/lib/toon-encoder.test.js.map +1 -0
- package/dist/lib/verification-events.d.ts +100 -0
- package/dist/lib/verification-events.d.ts.map +1 -0
- package/dist/lib/verification-events.js +162 -0
- package/dist/lib/verification-events.js.map +1 -0
- package/dist/lib/verification-events.test.d.ts +5 -0
- package/dist/lib/verification-events.test.d.ts.map +1 -0
- package/dist/lib/verification-events.test.js +193 -0
- package/dist/lib/verification-events.test.js.map +1 -0
- package/dist/server.d.ts +5 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +79 -21
- package/dist/server.js.map +1 -1
- package/dist/server.test.js +30 -0
- package/dist/server.test.js.map +1 -1
- package/dist/test-helpers/env-utils.d.ts +22 -0
- package/dist/test-helpers/env-utils.d.ts.map +1 -1
- package/dist/test-helpers/env-utils.js +38 -0
- package/dist/test-helpers/env-utils.js.map +1 -1
- package/dist/test-helpers/fuzz-generators.d.ts +58 -0
- package/dist/test-helpers/fuzz-generators.d.ts.map +1 -0
- package/dist/test-helpers/fuzz-generators.js +216 -0
- package/dist/test-helpers/fuzz-generators.js.map +1 -0
- package/dist/test-helpers/index.d.ts +1 -0
- package/dist/test-helpers/index.d.ts.map +1 -1
- package/dist/test-helpers/index.js +2 -0
- package/dist/test-helpers/index.js.map +1 -1
- package/dist/test-helpers/memfs-utils.d.ts +181 -0
- package/dist/test-helpers/memfs-utils.d.ts.map +1 -0
- package/dist/test-helpers/memfs-utils.js +292 -0
- package/dist/test-helpers/memfs-utils.js.map +1 -0
- package/dist/test-helpers/memfs-utils.test.d.ts +5 -0
- package/dist/test-helpers/memfs-utils.test.d.ts.map +1 -0
- package/dist/test-helpers/memfs-utils.test.js +338 -0
- package/dist/test-helpers/memfs-utils.test.js.map +1 -0
- package/dist/test-helpers/race-condition-helpers.d.ts +85 -0
- package/dist/test-helpers/race-condition-helpers.d.ts.map +1 -0
- package/dist/test-helpers/race-condition-helpers.js +279 -0
- package/dist/test-helpers/race-condition-helpers.js.map +1 -0
- package/dist/test-helpers/test-data-builders.d.ts +40 -3
- package/dist/test-helpers/test-data-builders.d.ts.map +1 -1
- package/dist/test-helpers/test-data-builders.js +54 -5
- package/dist/test-helpers/test-data-builders.js.map +1 -1
- package/dist/test-helpers/tool-validators.d.ts.map +1 -1
- package/dist/test-helpers/tool-validators.js +16 -1
- package/dist/test-helpers/tool-validators.js.map +1 -1
- package/dist/tools/context-stats.d.ts.map +1 -1
- package/dist/tools/context-stats.js +6 -8
- package/dist/tools/context-stats.js.map +1 -1
- package/dist/tools/export-confident.d.ts +145 -0
- package/dist/tools/export-confident.d.ts.map +1 -0
- package/dist/tools/export-confident.js +134 -0
- package/dist/tools/export-confident.js.map +1 -0
- package/dist/tools/export-confident.test.d.ts +7 -0
- package/dist/tools/export-confident.test.d.ts.map +1 -0
- package/dist/tools/export-confident.test.js +332 -0
- package/dist/tools/export-confident.test.js.map +1 -0
- package/dist/tools/export-datadog.d.ts +160 -0
- package/dist/tools/export-datadog.d.ts.map +1 -0
- package/dist/tools/export-datadog.js +160 -0
- package/dist/tools/export-datadog.js.map +1 -0
- package/dist/tools/export-datadog.test.d.ts +8 -0
- package/dist/tools/export-datadog.test.d.ts.map +1 -0
- package/dist/tools/export-datadog.test.js +419 -0
- package/dist/tools/export-datadog.test.js.map +1 -0
- package/dist/tools/export-langfuse.d.ts +137 -0
- package/dist/tools/export-langfuse.d.ts.map +1 -0
- package/dist/tools/export-langfuse.js +131 -0
- package/dist/tools/export-langfuse.js.map +1 -0
- package/dist/tools/export-langfuse.test.d.ts +7 -0
- package/dist/tools/export-langfuse.test.d.ts.map +1 -0
- package/dist/tools/export-langfuse.test.js +303 -0
- package/dist/tools/export-langfuse.test.js.map +1 -0
- package/dist/tools/export-phoenix.d.ts +145 -0
- package/dist/tools/export-phoenix.d.ts.map +1 -0
- package/dist/tools/export-phoenix.js +135 -0
- package/dist/tools/export-phoenix.js.map +1 -0
- package/dist/tools/export-phoenix.test.d.ts +7 -0
- package/dist/tools/export-phoenix.test.d.ts.map +1 -0
- package/dist/tools/export-phoenix.test.js +316 -0
- package/dist/tools/export-phoenix.test.js.map +1 -0
- package/dist/tools/health-check.d.ts +26 -0
- package/dist/tools/health-check.d.ts.map +1 -1
- package/dist/tools/health-check.js +36 -7
- package/dist/tools/health-check.js.map +1 -1
- package/dist/tools/index.d.ts +6 -0
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +6 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/inject-evaluations.d.ts +1315 -0
- package/dist/tools/inject-evaluations.d.ts.map +1 -0
- package/dist/tools/inject-evaluations.js +121 -0
- package/dist/tools/inject-evaluations.js.map +1 -0
- package/dist/tools/inject-evaluations.test.d.ts +5 -0
- package/dist/tools/inject-evaluations.test.d.ts.map +1 -0
- package/dist/tools/inject-evaluations.test.js +359 -0
- package/dist/tools/inject-evaluations.test.js.map +1 -0
- package/dist/tools/query-evaluations.d.ts +25 -4
- package/dist/tools/query-evaluations.d.ts.map +1 -1
- package/dist/tools/query-evaluations.js +26 -2
- package/dist/tools/query-evaluations.js.map +1 -1
- package/dist/tools/query-evaluations.test.js +53 -46
- package/dist/tools/query-evaluations.test.js.map +1 -1
- package/dist/tools/query-llm-events.js +2 -2
- package/dist/tools/query-llm-events.js.map +1 -1
- package/dist/tools/query-llm-events.test.js +6 -3
- package/dist/tools/query-llm-events.test.js.map +1 -1
- package/dist/tools/query-logs.d.ts +8 -8
- package/dist/tools/query-logs.js +3 -3
- package/dist/tools/query-logs.js.map +1 -1
- package/dist/tools/query-metrics.d.ts +4 -4
- package/dist/tools/query-metrics.js +2 -2
- package/dist/tools/query-metrics.js.map +1 -1
- package/dist/tools/query-traces.d.ts +8 -8
- package/dist/tools/query-verifications.d.ts +111 -0
- package/dist/tools/query-verifications.d.ts.map +1 -0
- package/dist/tools/query-verifications.js +101 -0
- package/dist/tools/query-verifications.js.map +1 -0
- package/dist/tools/query-verifications.test.d.ts +5 -0
- package/dist/tools/query-verifications.test.d.ts.map +1 -0
- package/dist/tools/query-verifications.test.js +156 -0
- package/dist/tools/query-verifications.test.js.map +1 -0
- package/dist/types/evaluation-hooks.d.ts +176 -0
- package/dist/types/evaluation-hooks.d.ts.map +1 -0
- package/dist/types/evaluation-hooks.js +49 -0
- package/dist/types/evaluation-hooks.js.map +1 -0
- package/package.json +11 -2
|
@@ -34,7 +34,22 @@ export function validateToolDefinition(tool, expectedName, schema, handler, requ
|
|
|
34
34
|
it('should have description mentioning key features', () => {
|
|
35
35
|
const desc = tool.description.toLowerCase();
|
|
36
36
|
for (const term of requiredDescriptionTerms) {
|
|
37
|
-
|
|
37
|
+
// Use word boundary matching for flexible validation
|
|
38
|
+
// Matches 'trace', 'traces', 'tracing' when searching for 'trace'
|
|
39
|
+
// For terms starting with '.', use literal match (e.g., '.claudeignore')
|
|
40
|
+
const termLower = term.toLowerCase();
|
|
41
|
+
let found;
|
|
42
|
+
if (termLower.startsWith('.')) {
|
|
43
|
+
// Literal match for dotfiles/extensions
|
|
44
|
+
found = desc.includes(termLower);
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
// Escape regex special chars and use word boundary
|
|
48
|
+
const escaped = termLower.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
49
|
+
const wordBoundaryPattern = new RegExp(`\\b${escaped}`, 'i');
|
|
50
|
+
found = wordBoundaryPattern.test(desc);
|
|
51
|
+
}
|
|
52
|
+
assert.ok(found, `Description should include "${term}" (word boundary match)`);
|
|
38
53
|
}
|
|
39
54
|
});
|
|
40
55
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-validators.js","sourceRoot":"","sources":["../../src/test-helpers/tool-validators.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AAYtC;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB,CACpC,IAAoB,EACpB,YAAoB,EACpB,MAAe,EACf,OAAgB,EAChB,wBAAmC;IAEnC,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC5B,MAAM,CAAC,WAAW,CAAC,OAAO,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YACtD,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC5B,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxB,MAAM,CAAC,WAAW,CAAC,OAAO,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;YACpD,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,IAAI,wBAAwB,IAAI,wBAAwB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpE,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;gBACzD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;gBAC5C,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;oBAC5C,MAAM,CAAC,EAAE,
|
|
1
|
+
{"version":3,"file":"tool-validators.js","sourceRoot":"","sources":["../../src/test-helpers/tool-validators.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AAYtC;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB,CACpC,IAAoB,EACpB,YAAoB,EACpB,MAAe,EACf,OAAgB,EAChB,wBAAmC;IAEnC,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC5B,MAAM,CAAC,WAAW,CAAC,OAAO,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YACtD,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC5B,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxB,MAAM,CAAC,WAAW,CAAC,OAAO,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;YACpD,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,IAAI,wBAAwB,IAAI,wBAAwB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpE,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;gBACzD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;gBAC5C,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;oBAC5C,qDAAqD;oBACrD,kEAAkE;oBAClE,yEAAyE;oBACzE,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;oBACrC,IAAI,KAAc,CAAC;oBACnB,IAAI,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC9B,wCAAwC;wBACxC,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;oBACnC,CAAC;yBAAM,CAAC;wBACN,mDAAmD;wBACnD,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;wBACjE,MAAM,mBAAmB,GAAG,IAAI,MAAM,CAAC,MAAM,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC;wBAC7D,KAAK,GAAG,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACzC,CAAC;oBACD,MAAM,CAAC,EAAE,CACP,KAAK,EACL,+BAA+B,IAAI,yBAAyB,CAC7D,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CACnC,OAAgB,EAChB,WAAmB;IAEnB,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;YAC5B,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;YACnB,MAAM,CAAC,WAAW,CAAC,OAAO,OAAO,EAAE,UAAU,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context-stats.d.ts","sourceRoot":"","sources":["../../src/tools/context-stats.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"context-stats.d.ts","sourceRoot":"","sources":["../../src/tools/context-stats.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAexB,eAAO,MAAM,kBAAkB;;;;;;;;;;;;EAI7B,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAsJnE,wBAAsB,YAAY,CAAC,QAAQ,EAAE,iBAAiB;eAqC/C,MAAM;aACR,MAAM;YACP,MAAM;aACL;QACP,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,gBAAgB,EAAE,MAAM,CAAC;KAC1B;YACO;QACN,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,aAAa,EAAE,MAAM,CAAC;KACvB;cACS,MAAM;WACT;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE;gBAC7D;QACV,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;KACf;cACS;QACR,aAAa,EAAE,MAAM,CAAC;QACtB,cAAc,EAAE,MAAM,CAAC;QACvB,UAAU,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,SAAS,EAAE,MAAM,CAAC;YAAC,YAAY,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAC9E;;;;;;;GAiEJ;AAED,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;CAK5B,CAAC"}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import { z } from 'zod';
|
|
5
5
|
import { existsSync, readFileSync, readdirSync, statSync } from 'fs';
|
|
6
6
|
import { join } from 'path';
|
|
7
|
-
import { CLAUDE_HOME, CONTEXT_WINDOW_SIZE, MODEL_PRICING, checkPricingStaleness } from '../lib/constants.js';
|
|
7
|
+
import { CLAUDE_HOME, CONTEXT_WINDOW_SIZE, MODEL_PRICING, checkPricingStaleness, SYSTEM_PROMPT_TOKENS, SYSTEM_TOOLS_TOKENS, HISTORY_RECENT_DAYS, } from '../lib/constants.js';
|
|
8
8
|
import { loadJsonSafe } from '../lib/file-utils.js';
|
|
9
9
|
export const contextStatsSchema = z.object({
|
|
10
10
|
sessionId: z.string().optional().describe('Session ID (default: most recent)'),
|
|
@@ -181,15 +181,13 @@ export async function contextStats(rawInput) {
|
|
|
181
181
|
}
|
|
182
182
|
if (input.includeBreakdown) {
|
|
183
183
|
// Estimate breakdown based on typical overhead
|
|
184
|
-
const
|
|
185
|
-
const systemTools = 15000;
|
|
186
|
-
const overhead = systemPrompt + systemTools;
|
|
184
|
+
const overhead = SYSTEM_PROMPT_TOKENS + SYSTEM_TOOLS_TOKENS;
|
|
187
185
|
const messages = Math.max(0, usage.totalContextTokens - overhead);
|
|
188
186
|
result.breakdown = {
|
|
189
|
-
systemPrompt,
|
|
190
|
-
systemTools,
|
|
187
|
+
systemPrompt: SYSTEM_PROMPT_TOKENS,
|
|
188
|
+
systemTools: SYSTEM_TOOLS_TOKENS,
|
|
191
189
|
messages,
|
|
192
|
-
other: Math.max(0, usage.totalContextTokens -
|
|
190
|
+
other: Math.max(0, usage.totalContextTokens - SYSTEM_PROMPT_TOKENS - SYSTEM_TOOLS_TOKENS - messages),
|
|
193
191
|
};
|
|
194
192
|
}
|
|
195
193
|
// Load historical data
|
|
@@ -200,7 +198,7 @@ export async function contextStats(rawInput) {
|
|
|
200
198
|
result.history = {
|
|
201
199
|
totalSessions: history.sessions.length,
|
|
202
200
|
avgUtilization: Math.round(avgUtilization * 10) / 10,
|
|
203
|
-
recentDays: history.dailyAverages.slice(-
|
|
201
|
+
recentDays: history.dailyAverages.slice(-HISTORY_RECENT_DAYS).map(d => ({
|
|
204
202
|
date: d.date,
|
|
205
203
|
avgTokens: Math.round(d.avgTokens),
|
|
206
204
|
sessionCount: d.sessionCount,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context-stats.js","sourceRoot":"","sources":["../../src/tools/context-stats.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AACrE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,
|
|
1
|
+
{"version":3,"file":"context-stats.js","sourceRoot":"","sources":["../../src/tools/context-stats.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AACrE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EACL,WAAW,EACX,mBAAmB,EAEnB,aAAa,EACb,qBAAqB,EACrB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IAC9E,gBAAgB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IAC/E,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,uBAAuB,CAAC;CACzE,CAAC,CAAC;AAkCH;;GAEG;AACH,SAAS,iBAAiB;IACxB,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;IAClD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1C,IAAI,UAAU,GAAsF,IAAI,CAAC;IAEzG,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;QAC7C,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;YAClC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YAC/C,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC;YACnC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;gBAAE,SAAS;YAElC,MAAM,QAAQ,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC5E,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;gBAC5C,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACvC,IAAI,CAAC,UAAU,IAAI,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC;oBAClE,UAAU,GAAG;wBACX,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;wBACxC,UAAU,EAAE,OAAO;wBACnB,QAAQ;wBACR,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE;qBACnC,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACnI,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAgB;IACzC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEzC,IAAI,WAAW,GAAwB,IAAI,CAAC;QAC5C,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,KAAyB,CAAC;QAE9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAE/B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBACxD,YAAY,EAAE,CAAC;gBACjB,CAAC;gBAED,IAAI,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;oBACzB,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;gBAC9B,CAAC;gBAED,IAAI,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;oBACzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;oBAClC,MAAM,WAAW,GAAG,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC;oBAC5C,MAAM,YAAY,GAAG,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC;oBAC9C,MAAM,eAAe,GAAG,KAAK,CAAC,uBAAuB,IAAI,CAAC,CAAC;oBAC3D,MAAM,mBAAmB,GAAG,KAAK,CAAC,2BAA2B,IAAI,CAAC,CAAC;oBAEnE,MAAM,kBAAkB,GAAG,WAAW,GAAG,eAAe,GAAG,mBAAmB,CAAC;oBAC/E,MAAM,kBAAkB,GAAG,CAAC,kBAAkB,GAAG,mBAAmB,CAAC,GAAG,GAAG,CAAC;oBAC5E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,mBAAmB,GAAG,kBAAkB,CAAC,CAAC;oBAExE,WAAW,GAAG;wBACZ,WAAW;wBACX,YAAY;wBACZ,eAAe;wBACf,mBAAmB;wBACnB,kBAAkB;wBAClB,kBAAkB,EAAE,IAAI,CAAC,KAAK,CAAC,kBAAkB,GAAG,EAAE,CAAC,GAAG,EAAE;wBAC5D,SAAS;wBACT,YAAY;wBACZ,KAAK;qBACN,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,uBAAuB;YACzB,CAAC;QACH,CAAC;QAED,IAAI,WAAW,EAAE,CAAC;YAChB,WAAW,CAAC,YAAY,GAAG,YAAY,CAAC;QAC1C,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,KAAmB;IACxC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,0BAA0B,CAAC;IACxD,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAErE,MAAM,SAAS,GAAG,CAAC,KAAK,CAAC,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC;IAClE,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;IAErE,OAAO;QACL,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,GAAG,KAAK;QAChD,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,GAAG,KAAK;QAClD,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK;KAChE,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAA2B;IAC5D,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACjD,eAAe;IACf,IAAI,WAAW,GAAuE,IAAI,CAAC;IAE3F,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACpB,8BAA8B;QAC9B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QAClD,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC5B,MAAM,WAAW,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;gBAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC,SAAS,QAAQ,CAAC,CAAC;gBACxE,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACzB,WAAW,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;oBAC5E,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,WAAW,GAAG,iBAAiB,EAAE,CAAC;IACpC,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,sBAAsB,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,mBAAmB;SACvF,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,iBAAiB,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IACtD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO;YACL,SAAS,EAAE,WAAW,CAAC,SAAS;YAChC,KAAK,EAAE,oCAAoC;SAC5C,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GA6BR;QACF,SAAS,EAAE,WAAW,CAAC,SAAS;QAChC,OAAO,EAAE,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;QAClD,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,OAAO,EAAE;YACP,WAAW,EAAE,KAAK,CAAC,kBAAkB;YACrC,WAAW,EAAE,GAAG,KAAK,CAAC,kBAAkB,GAAG;YAC3C,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,gBAAgB,EAAE,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;SACpE;QACD,MAAM,EAAE;YACN,KAAK,EAAE,KAAK,CAAC,WAAW;YACxB,MAAM,EAAE,KAAK,CAAC,YAAY;YAC1B,SAAS,EAAE,KAAK,CAAC,eAAe;YAChC,aAAa,EAAE,KAAK,CAAC,mBAAmB;SACzC;QACD,QAAQ,EAAE,KAAK,CAAC,YAAY;KAC7B,CAAC;IAEF,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,SAAS,GAAG,qBAAqB,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,GAAG;YACZ,KAAK,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YACtC,MAAM,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YACxC,KAAK,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YACtC,GAAG,CAAC,SAAS,CAAC,OAAO,IAAI;gBACvB,OAAO,EAAE,mBAAmB,SAAS,CAAC,eAAe,qCAAqC;aAC3F,CAAC;SACH,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC3B,+CAA+C;QAC/C,MAAM,QAAQ,GAAG,oBAAoB,GAAG,mBAAmB,CAAC;QAC5D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,kBAAkB,GAAG,QAAQ,CAAC,CAAC;QAElE,MAAM,CAAC,SAAS,GAAG;YACjB,YAAY,EAAE,oBAAoB;YAClC,WAAW,EAAE,mBAAmB;YAChC,QAAQ;YACR,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,kBAAkB,GAAG,oBAAoB,GAAG,mBAAmB,GAAG,QAAQ,CAAC;SACrG,CAAC;IACJ,CAAC;IAED,uBAAuB;IACvB,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,EAAE,sBAAsB,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,YAAY,CAAiB,WAAW,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;IAE3G,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAEpH,MAAM,CAAC,OAAO,GAAG;YACf,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,MAAM;YACtC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,EAAE,CAAC,GAAG,EAAE;YACpD,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,mBAAmB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACtE,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;gBAClC,YAAY,EAAE,CAAC,CAAC,YAAY;aAC7B,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,IAAI,EAAE,mBAAmB;IACzB,WAAW,EAAE,kEAAkE;IAC/E,WAAW,EAAE,kBAAkB;IAC/B,OAAO,EAAE,YAAY;CACtB,CAAC"}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Export evaluations to Confident AI via OTLP HTTP
|
|
3
|
+
*
|
|
4
|
+
* MCP tool for exporting gen_ai.evaluation.result events to Confident AI
|
|
5
|
+
* for LLM evaluation analysis and metric collection.
|
|
6
|
+
*/
|
|
7
|
+
import { z } from 'zod';
|
|
8
|
+
import { type ConfidentConfig } from '../lib/confident-export.js';
|
|
9
|
+
import type { TelemetryBackend } from '../backends/index.js';
|
|
10
|
+
export declare const exportConfidentSchema: z.ZodObject<{
|
|
11
|
+
limit: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
12
|
+
batchSize: z.ZodOptional<z.ZodNumber>;
|
|
13
|
+
dryRun: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
14
|
+
metricCollection: z.ZodOptional<z.ZodString>;
|
|
15
|
+
environment: z.ZodOptional<z.ZodEnum<["production", "development", "staging", "testing"]>>;
|
|
16
|
+
endpoint: z.ZodOptional<z.ZodString>;
|
|
17
|
+
apiKey: z.ZodOptional<z.ZodString>;
|
|
18
|
+
startDate: z.ZodOptional<z.ZodString>;
|
|
19
|
+
endDate: z.ZodOptional<z.ZodString>;
|
|
20
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
21
|
+
traceId: z.ZodOptional<z.ZodString>;
|
|
22
|
+
evaluationName: z.ZodOptional<z.ZodString>;
|
|
23
|
+
scoreMin: z.ZodOptional<z.ZodNumber>;
|
|
24
|
+
scoreMax: z.ZodOptional<z.ZodNumber>;
|
|
25
|
+
scoreLabel: z.ZodOptional<z.ZodString>;
|
|
26
|
+
evaluatorType: z.ZodOptional<z.ZodEnum<["llm", "human", "rule", "classifier"]>>;
|
|
27
|
+
}, "strip", z.ZodTypeAny, {
|
|
28
|
+
limit: number;
|
|
29
|
+
dryRun: boolean;
|
|
30
|
+
traceId?: string | undefined;
|
|
31
|
+
sessionId?: string | undefined;
|
|
32
|
+
evaluationName?: string | undefined;
|
|
33
|
+
scoreLabel?: string | undefined;
|
|
34
|
+
scoreMin?: number | undefined;
|
|
35
|
+
scoreMax?: number | undefined;
|
|
36
|
+
evaluatorType?: "llm" | "human" | "rule" | "classifier" | undefined;
|
|
37
|
+
endpoint?: string | undefined;
|
|
38
|
+
endDate?: string | undefined;
|
|
39
|
+
startDate?: string | undefined;
|
|
40
|
+
batchSize?: number | undefined;
|
|
41
|
+
apiKey?: string | undefined;
|
|
42
|
+
environment?: "production" | "development" | "staging" | "testing" | undefined;
|
|
43
|
+
metricCollection?: string | undefined;
|
|
44
|
+
}, {
|
|
45
|
+
limit?: number | undefined;
|
|
46
|
+
traceId?: string | undefined;
|
|
47
|
+
sessionId?: string | undefined;
|
|
48
|
+
evaluationName?: string | undefined;
|
|
49
|
+
scoreLabel?: string | undefined;
|
|
50
|
+
scoreMin?: number | undefined;
|
|
51
|
+
scoreMax?: number | undefined;
|
|
52
|
+
evaluatorType?: "llm" | "human" | "rule" | "classifier" | undefined;
|
|
53
|
+
endpoint?: string | undefined;
|
|
54
|
+
endDate?: string | undefined;
|
|
55
|
+
startDate?: string | undefined;
|
|
56
|
+
dryRun?: boolean | undefined;
|
|
57
|
+
batchSize?: number | undefined;
|
|
58
|
+
apiKey?: string | undefined;
|
|
59
|
+
environment?: "production" | "development" | "staging" | "testing" | undefined;
|
|
60
|
+
metricCollection?: string | undefined;
|
|
61
|
+
}>;
|
|
62
|
+
export type ExportConfidentInput = z.input<typeof exportConfidentSchema>;
|
|
63
|
+
/** Response from export operation */
|
|
64
|
+
export interface ExportConfidentResponse {
|
|
65
|
+
success: boolean;
|
|
66
|
+
dryRun: boolean;
|
|
67
|
+
evaluationsFound: number;
|
|
68
|
+
evaluationsExported: number;
|
|
69
|
+
batches: number;
|
|
70
|
+
failed: number;
|
|
71
|
+
durationMs: number;
|
|
72
|
+
errors?: string[];
|
|
73
|
+
preview?: Array<{
|
|
74
|
+
traceId?: string;
|
|
75
|
+
evaluationName: string;
|
|
76
|
+
scoreValue?: number;
|
|
77
|
+
scoreLabel?: string;
|
|
78
|
+
}>;
|
|
79
|
+
/** Link to view in Confident AI Observatory */
|
|
80
|
+
observatoryUrl?: string;
|
|
81
|
+
}
|
|
82
|
+
export interface ExportConfidentOptions {
|
|
83
|
+
localBackend?: TelemetryBackend;
|
|
84
|
+
configOverrides?: Partial<ConfidentConfig>;
|
|
85
|
+
skipExport?: boolean;
|
|
86
|
+
}
|
|
87
|
+
export declare function exportConfident(rawInput: ExportConfidentInput, options?: ExportConfidentOptions): Promise<ExportConfidentResponse>;
|
|
88
|
+
export declare const exportConfidentTool: {
|
|
89
|
+
name: string;
|
|
90
|
+
description: string;
|
|
91
|
+
inputSchema: z.ZodObject<{
|
|
92
|
+
limit: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
93
|
+
batchSize: z.ZodOptional<z.ZodNumber>;
|
|
94
|
+
dryRun: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
95
|
+
metricCollection: z.ZodOptional<z.ZodString>;
|
|
96
|
+
environment: z.ZodOptional<z.ZodEnum<["production", "development", "staging", "testing"]>>;
|
|
97
|
+
endpoint: z.ZodOptional<z.ZodString>;
|
|
98
|
+
apiKey: z.ZodOptional<z.ZodString>;
|
|
99
|
+
startDate: z.ZodOptional<z.ZodString>;
|
|
100
|
+
endDate: z.ZodOptional<z.ZodString>;
|
|
101
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
102
|
+
traceId: z.ZodOptional<z.ZodString>;
|
|
103
|
+
evaluationName: z.ZodOptional<z.ZodString>;
|
|
104
|
+
scoreMin: z.ZodOptional<z.ZodNumber>;
|
|
105
|
+
scoreMax: z.ZodOptional<z.ZodNumber>;
|
|
106
|
+
scoreLabel: z.ZodOptional<z.ZodString>;
|
|
107
|
+
evaluatorType: z.ZodOptional<z.ZodEnum<["llm", "human", "rule", "classifier"]>>;
|
|
108
|
+
}, "strip", z.ZodTypeAny, {
|
|
109
|
+
limit: number;
|
|
110
|
+
dryRun: boolean;
|
|
111
|
+
traceId?: string | undefined;
|
|
112
|
+
sessionId?: string | undefined;
|
|
113
|
+
evaluationName?: string | undefined;
|
|
114
|
+
scoreLabel?: string | undefined;
|
|
115
|
+
scoreMin?: number | undefined;
|
|
116
|
+
scoreMax?: number | undefined;
|
|
117
|
+
evaluatorType?: "llm" | "human" | "rule" | "classifier" | undefined;
|
|
118
|
+
endpoint?: string | undefined;
|
|
119
|
+
endDate?: string | undefined;
|
|
120
|
+
startDate?: string | undefined;
|
|
121
|
+
batchSize?: number | undefined;
|
|
122
|
+
apiKey?: string | undefined;
|
|
123
|
+
environment?: "production" | "development" | "staging" | "testing" | undefined;
|
|
124
|
+
metricCollection?: string | undefined;
|
|
125
|
+
}, {
|
|
126
|
+
limit?: number | undefined;
|
|
127
|
+
traceId?: string | undefined;
|
|
128
|
+
sessionId?: string | undefined;
|
|
129
|
+
evaluationName?: string | undefined;
|
|
130
|
+
scoreLabel?: string | undefined;
|
|
131
|
+
scoreMin?: number | undefined;
|
|
132
|
+
scoreMax?: number | undefined;
|
|
133
|
+
evaluatorType?: "llm" | "human" | "rule" | "classifier" | undefined;
|
|
134
|
+
endpoint?: string | undefined;
|
|
135
|
+
endDate?: string | undefined;
|
|
136
|
+
startDate?: string | undefined;
|
|
137
|
+
dryRun?: boolean | undefined;
|
|
138
|
+
batchSize?: number | undefined;
|
|
139
|
+
apiKey?: string | undefined;
|
|
140
|
+
environment?: "production" | "development" | "staging" | "testing" | undefined;
|
|
141
|
+
metricCollection?: string | undefined;
|
|
142
|
+
}>;
|
|
143
|
+
handler: typeof exportConfident;
|
|
144
|
+
};
|
|
145
|
+
//# sourceMappingURL=export-confident.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"export-confident.d.ts","sourceRoot":"","sources":["../../src/tools/export-confident.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAGL,KAAK,eAAe,EACrB,MAAM,4BAA4B,CAAC;AAWpC,OAAO,KAAK,EAAiB,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAG5E,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAuBhC,CAAC;AAEH,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEzE,qCAAqC;AACrC,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,cAAc,EAAE,MAAM,CAAC;QACvB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC,CAAC;IACH,+CAA+C;IAC/C,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAGD,MAAM,WAAW,sBAAsB;IACrC,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,eAAe,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IAC3C,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,wBAAsB,eAAe,CACnC,QAAQ,EAAE,oBAAoB,EAC9B,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,uBAAuB,CAAC,CAuGlC;AAED,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAK/B,CAAC"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Export evaluations to Confident AI via OTLP HTTP
|
|
3
|
+
*
|
|
4
|
+
* MCP tool for exporting gen_ai.evaluation.result events to Confident AI
|
|
5
|
+
* for LLM evaluation analysis and metric collection.
|
|
6
|
+
*/
|
|
7
|
+
import { z } from 'zod';
|
|
8
|
+
import { MultiDirectoryBackend } from '../backends/local-jsonl.js';
|
|
9
|
+
import { validateConfidentConfig, exportToConfident, } from '../lib/confident-export.js';
|
|
10
|
+
import { validateLimit, validateDateRange, InputValidationError, } from '../lib/input-validator.js';
|
|
11
|
+
import { sessionSchema, traceIdSchema, dateRangeSchema, } from '../lib/shared-schemas.js';
|
|
12
|
+
import { MAX_EXPORT_LIMIT, EXPORT_PREVIEW_LIMIT } from '../lib/constants.js';
|
|
13
|
+
export const exportConfidentSchema = z.object({
|
|
14
|
+
// Filters (consistent with Langfuse)
|
|
15
|
+
evaluationName: z.string().optional().describe('Evaluation name (substring)'),
|
|
16
|
+
scoreMin: z.number().optional().describe('Min score threshold'),
|
|
17
|
+
scoreMax: z.number().optional().describe('Max score threshold'),
|
|
18
|
+
scoreLabel: z.string().optional().describe('Score label (exact)'),
|
|
19
|
+
evaluatorType: z.enum(['llm', 'human', 'rule', 'classifier']).optional().describe('Evaluator type'),
|
|
20
|
+
...traceIdSchema,
|
|
21
|
+
...sessionSchema,
|
|
22
|
+
...dateRangeSchema,
|
|
23
|
+
// Export options (consistent with Langfuse)
|
|
24
|
+
limit: z.number().max(MAX_EXPORT_LIMIT).optional().default(1000).describe('Max evaluations to export (max 10000)'),
|
|
25
|
+
batchSize: z.number().min(1).max(1000).optional().describe('Batch size for export (default 100)'),
|
|
26
|
+
dryRun: z.boolean().optional().default(false).describe('Preview export without sending'),
|
|
27
|
+
// Confident AI specific
|
|
28
|
+
metricCollection: z.string().optional().describe('Metric collection for online eval'),
|
|
29
|
+
environment: z.enum(['production', 'development', 'staging', 'testing']).optional().describe('Environment tag'),
|
|
30
|
+
// Config overrides
|
|
31
|
+
endpoint: z.string().optional().describe('Override CONFIDENT_ENDPOINT'),
|
|
32
|
+
apiKey: z.string().optional().describe('Override CONFIDENT_API_KEY'),
|
|
33
|
+
});
|
|
34
|
+
export async function exportConfident(rawInput, options) {
|
|
35
|
+
try {
|
|
36
|
+
const input = exportConfidentSchema.parse(rawInput);
|
|
37
|
+
// Validate inputs
|
|
38
|
+
const validatedLimit = Math.min(validateLimit(input.limit), MAX_EXPORT_LIMIT);
|
|
39
|
+
// Validate date range if provided
|
|
40
|
+
if (input.startDate || input.endDate) {
|
|
41
|
+
validateDateRange(input.startDate, input.endDate);
|
|
42
|
+
}
|
|
43
|
+
// Validate score range
|
|
44
|
+
if (input.scoreMin !== undefined && input.scoreMax !== undefined) {
|
|
45
|
+
if (input.scoreMin > input.scoreMax) {
|
|
46
|
+
throw new Error(`Invalid score range: scoreMin (${input.scoreMin}) cannot exceed scoreMax (${input.scoreMax})`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
// Build query options
|
|
50
|
+
const queryOptions = {
|
|
51
|
+
evaluationName: input.evaluationName,
|
|
52
|
+
scoreMin: input.scoreMin,
|
|
53
|
+
scoreMax: input.scoreMax,
|
|
54
|
+
scoreLabel: input.scoreLabel,
|
|
55
|
+
evaluatorType: input.evaluatorType,
|
|
56
|
+
traceId: input.traceId,
|
|
57
|
+
sessionId: input.sessionId,
|
|
58
|
+
startDate: input.startDate,
|
|
59
|
+
endDate: input.endDate,
|
|
60
|
+
limit: validatedLimit,
|
|
61
|
+
};
|
|
62
|
+
// Query evaluations from local backend
|
|
63
|
+
const local = options?.localBackend ?? new MultiDirectoryBackend();
|
|
64
|
+
const evaluations = await local.queryEvaluations(queryOptions);
|
|
65
|
+
// Handle dry run
|
|
66
|
+
if (input.dryRun) {
|
|
67
|
+
return {
|
|
68
|
+
success: true,
|
|
69
|
+
dryRun: true,
|
|
70
|
+
evaluationsFound: evaluations.length,
|
|
71
|
+
evaluationsExported: 0,
|
|
72
|
+
batches: 0,
|
|
73
|
+
failed: 0,
|
|
74
|
+
durationMs: 0,
|
|
75
|
+
preview: evaluations.slice(0, EXPORT_PREVIEW_LIMIT).map(e => ({
|
|
76
|
+
traceId: e.traceId,
|
|
77
|
+
evaluationName: e.evaluationName,
|
|
78
|
+
scoreValue: e.scoreValue,
|
|
79
|
+
scoreLabel: e.scoreLabel,
|
|
80
|
+
})),
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
// Validate Confident AI config (will throw if invalid)
|
|
84
|
+
// Optimize batch size - don't use larger batch than evaluation count
|
|
85
|
+
const optimalBatchSize = input.batchSize
|
|
86
|
+
? Math.min(input.batchSize, evaluations.length || 1)
|
|
87
|
+
: undefined;
|
|
88
|
+
const config = validateConfidentConfig({
|
|
89
|
+
endpoint: input.endpoint,
|
|
90
|
+
apiKey: input.apiKey,
|
|
91
|
+
batchSize: optimalBatchSize,
|
|
92
|
+
environment: input.environment,
|
|
93
|
+
metricCollection: input.metricCollection,
|
|
94
|
+
...options?.configOverrides,
|
|
95
|
+
});
|
|
96
|
+
// Skip actual export if requested (for testing)
|
|
97
|
+
if (options?.skipExport) {
|
|
98
|
+
return {
|
|
99
|
+
success: true,
|
|
100
|
+
dryRun: false,
|
|
101
|
+
evaluationsFound: evaluations.length,
|
|
102
|
+
evaluationsExported: evaluations.length,
|
|
103
|
+
batches: Math.ceil(evaluations.length / config.batchSize),
|
|
104
|
+
failed: 0,
|
|
105
|
+
durationMs: 0,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
// Export to Confident AI
|
|
109
|
+
const result = await exportToConfident(evaluations, config);
|
|
110
|
+
return {
|
|
111
|
+
success: result.success,
|
|
112
|
+
dryRun: false,
|
|
113
|
+
evaluationsFound: evaluations.length,
|
|
114
|
+
evaluationsExported: result.evaluationsExported,
|
|
115
|
+
batches: result.batches,
|
|
116
|
+
failed: result.failed,
|
|
117
|
+
durationMs: result.durationMs,
|
|
118
|
+
...(result.errors && { errors: result.errors }),
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
catch (error) {
|
|
122
|
+
if (error instanceof InputValidationError) {
|
|
123
|
+
throw new Error(`Validation error (${error.field}): ${error.message}`);
|
|
124
|
+
}
|
|
125
|
+
throw error;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
export const exportConfidentTool = {
|
|
129
|
+
name: 'obs_export_confident',
|
|
130
|
+
description: 'Export evaluation results to Confident AI via OTLP HTTP for LLM evaluation and metric collection.',
|
|
131
|
+
inputSchema: exportConfidentSchema,
|
|
132
|
+
handler: exportConfident,
|
|
133
|
+
};
|
|
134
|
+
//# sourceMappingURL=export-confident.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"export-confident.js","sourceRoot":"","sources":["../../src/tools/export-confident.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EACL,uBAAuB,EACvB,iBAAiB,GAElB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,aAAa,EACb,aAAa,EACb,eAAe,GAChB,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAE7E,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,qCAAqC;IACrC,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IAC7E,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;IAC/D,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;IAC/D,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;IACjE,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IACnG,GAAG,aAAa;IAChB,GAAG,aAAa;IAChB,GAAG,eAAe;IAElB,4CAA4C;IAC5C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,uCAAuC,CAAC;IAClH,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IACjG,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,gCAAgC,CAAC;IAExF,wBAAwB;IACxB,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IACrF,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAE/G,mBAAmB;IACnB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IACvE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;CACrE,CAAC,CAAC;AA+BH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,QAA8B,EAC9B,OAAgC;IAEhC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,qBAAqB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAEpD,kBAAkB;QAClB,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAE9E,kCAAkC;QAClC,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YACrC,iBAAiB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACpD,CAAC;QAED,uBAAuB;QACvB,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YACjE,IAAI,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,kCAAkC,KAAK,CAAC,QAAQ,6BAA6B,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC;YAClH,CAAC;QACH,CAAC;QAED,sBAAsB;QACtB,MAAM,YAAY,GAAG;YACnB,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,aAAa,EAAE,KAAK,CAAC,aAA0C;YAC/D,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,cAAc;SACtB,CAAC;QAEF,uCAAuC;QACvC,MAAM,KAAK,GAAG,OAAO,EAAE,YAAY,IAAI,IAAI,qBAAqB,EAAE,CAAC;QACnE,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,gBAAiB,CAAC,YAAY,CAAC,CAAC;QAEhE,iBAAiB;QACjB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,IAAI;gBACZ,gBAAgB,EAAE,WAAW,CAAC,MAAM;gBACpC,mBAAmB,EAAE,CAAC;gBACtB,OAAO,EAAE,CAAC;gBACV,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,CAAC;gBACb,OAAO,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC5D,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,cAAc,EAAE,CAAC,CAAC,cAAc;oBAChC,UAAU,EAAE,CAAC,CAAC,UAAU;oBACxB,UAAU,EAAE,CAAC,CAAC,UAAU;iBACzB,CAAC,CAAC;aACJ,CAAC;QACJ,CAAC;QAED,uDAAuD;QACvD,qEAAqE;QACrE,MAAM,gBAAgB,GAAG,KAAK,CAAC,SAAS;YACtC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,IAAI,CAAC,CAAC;YACpD,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,MAAM,GAAG,uBAAuB,CAAC;YACrC,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,SAAS,EAAE,gBAAgB;YAC3B,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YACxC,GAAG,OAAO,EAAE,eAAe;SAC5B,CAAC,CAAC;QAEH,gDAAgD;QAChD,IAAI,OAAO,EAAE,UAAU,EAAE,CAAC;YACxB,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,KAAK;gBACb,gBAAgB,EAAE,WAAW,CAAC,MAAM;gBACpC,mBAAmB,EAAE,WAAW,CAAC,MAAM;gBACvC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC;gBACzD,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,CAAC;aACd,CAAC;QACJ,CAAC;QAED,yBAAyB;QACzB,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAE5D,OAAO;YACL,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM,EAAE,KAAK;YACb,gBAAgB,EAAE,WAAW,CAAC,MAAM;YACpC,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;YAC/C,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;SAChD,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,oBAAoB,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,qBAAqB,KAAK,CAAC,KAAK,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,IAAI,EAAE,sBAAsB;IAC5B,WAAW,EAAE,mGAAmG;IAChH,WAAW,EAAE,qBAAqB;IAClC,OAAO,EAAE,eAAe;CACzB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"export-confident.test.d.ts","sourceRoot":"","sources":["../../src/tools/export-confident.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|