judgeval 0.2.9 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/constants.d.ts +2 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +2 -0
- package/dist/constants.js.map +1 -0
- package/dist/data/example-params.d.ts +13 -0
- package/dist/data/example-params.d.ts.map +1 -0
- package/dist/data/example-params.js +12 -0
- package/dist/data/example-params.js.map +1 -0
- package/dist/data/example.d.ts +4 -0
- package/dist/data/example.d.ts.map +1 -0
- package/dist/data/example.js +10 -0
- package/dist/data/example.js.map +1 -0
- package/dist/data/index.d.ts +3 -0
- package/dist/data/index.d.ts.map +1 -0
- package/dist/data/index.js +3 -0
- package/dist/data/index.js.map +1 -0
- package/dist/env.d.ts +6 -0
- package/dist/env.d.ts.map +1 -0
- package/dist/env.js +13 -0
- package/dist/env.js.map +1 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +2 -0
- package/dist/index.mjs.map +1 -0
- package/dist/index.umd.js +2 -0
- package/dist/index.umd.js.map +1 -0
- package/dist/internal/api/index.d.ts +23 -0
- package/dist/internal/api/index.d.ts.map +1 -0
- package/dist/internal/api/index.js +132 -0
- package/dist/internal/api/index.js.map +1 -0
- package/dist/internal/api/models/BaseScorer.d.ts +23 -0
- package/dist/internal/api/models/BaseScorer.d.ts.map +1 -0
- package/dist/internal/api/models/BaseScorer.js +6 -0
- package/dist/internal/api/models/BaseScorer.js.map +1 -0
- package/dist/internal/api/models/EvalResults.d.ts +12 -0
- package/dist/internal/api/models/EvalResults.d.ts.map +1 -0
- package/dist/internal/api/models/EvalResults.js +6 -0
- package/dist/internal/api/models/EvalResults.js.map +1 -0
- package/dist/internal/api/models/EvalResultsFetch.d.ts +9 -0
- package/dist/internal/api/models/EvalResultsFetch.d.ts.map +1 -0
- package/dist/internal/api/models/EvalResultsFetch.js +6 -0
- package/dist/internal/api/models/EvalResultsFetch.js.map +1 -0
- package/dist/internal/api/models/Example.d.ts +10 -0
- package/dist/internal/api/models/Example.d.ts.map +1 -0
- package/dist/internal/api/models/Example.js +6 -0
- package/dist/internal/api/models/Example.js.map +1 -0
- package/dist/internal/api/models/ExampleEvaluationRun.d.ts +20 -0
- package/dist/internal/api/models/ExampleEvaluationRun.d.ts.map +1 -0
- package/dist/internal/api/models/ExampleEvaluationRun.js +6 -0
- package/dist/internal/api/models/ExampleEvaluationRun.js.map +1 -0
- package/dist/internal/api/models/FetchPromptScorersRequest.d.ts +8 -0
- package/dist/internal/api/models/FetchPromptScorersRequest.d.ts.map +1 -0
- package/dist/internal/api/models/FetchPromptScorersRequest.js +6 -0
- package/dist/internal/api/models/FetchPromptScorersRequest.js.map +1 -0
- package/dist/internal/api/models/FetchPromptScorersResponse.d.ts +9 -0
- package/dist/internal/api/models/FetchPromptScorersResponse.d.ts.map +1 -0
- package/dist/internal/api/models/FetchPromptScorersResponse.js +6 -0
- package/dist/internal/api/models/FetchPromptScorersResponse.js.map +1 -0
- package/dist/internal/api/models/OtelTraceSpan.d.ts +33 -0
- package/dist/internal/api/models/OtelTraceSpan.d.ts.map +1 -0
- package/dist/internal/api/models/OtelTraceSpan.js +6 -0
- package/dist/internal/api/models/OtelTraceSpan.js.map +1 -0
- package/dist/internal/api/models/PromptScorer.d.ts +14 -0
- package/dist/internal/api/models/PromptScorer.d.ts.map +1 -0
- package/dist/internal/api/models/PromptScorer.js +6 -0
- package/dist/internal/api/models/PromptScorer.js.map +1 -0
- package/dist/internal/api/models/ResolveProjectNameRequest.d.ts +8 -0
- package/dist/internal/api/models/ResolveProjectNameRequest.d.ts.map +1 -0
- package/dist/internal/api/models/ResolveProjectNameRequest.js +6 -0
- package/dist/internal/api/models/ResolveProjectNameRequest.js.map +1 -0
- package/dist/internal/api/models/ResolveProjectNameResponse.d.ts +8 -0
- package/dist/internal/api/models/ResolveProjectNameResponse.d.ts.map +1 -0
- package/dist/internal/api/models/ResolveProjectNameResponse.js +6 -0
- package/dist/internal/api/models/ResolveProjectNameResponse.js.map +1 -0
- package/dist/internal/api/models/SavePromptScorerRequest.d.ts +12 -0
- package/dist/internal/api/models/SavePromptScorerRequest.d.ts.map +1 -0
- package/dist/internal/api/models/SavePromptScorerRequest.js +6 -0
- package/dist/internal/api/models/SavePromptScorerRequest.js.map +1 -0
- package/dist/internal/api/models/SavePromptScorerResponse.d.ts +9 -0
- package/dist/internal/api/models/SavePromptScorerResponse.d.ts.map +1 -0
- package/dist/internal/api/models/SavePromptScorerResponse.js +6 -0
- package/dist/internal/api/models/SavePromptScorerResponse.js.map +1 -0
- package/dist/internal/api/models/ScorerConfig.d.ts +13 -0
- package/dist/internal/api/models/ScorerConfig.d.ts.map +1 -0
- package/dist/internal/api/models/ScorerConfig.js +6 -0
- package/dist/internal/api/models/ScorerConfig.js.map +1 -0
- package/dist/internal/api/models/ScorerData.d.ts +17 -0
- package/dist/internal/api/models/ScorerData.d.ts.map +1 -0
- package/dist/internal/api/models/ScorerData.js +6 -0
- package/dist/internal/api/models/ScorerData.js.map +1 -0
- package/dist/internal/api/models/ScorerExistsRequest.d.ts +8 -0
- package/dist/internal/api/models/ScorerExistsRequest.d.ts.map +1 -0
- package/dist/internal/api/models/ScorerExistsRequest.js +6 -0
- package/dist/internal/api/models/ScorerExistsRequest.js.map +1 -0
- package/dist/internal/api/models/ScorerExistsResponse.d.ts +8 -0
- package/dist/internal/api/models/ScorerExistsResponse.d.ts.map +1 -0
- package/dist/internal/api/models/ScorerExistsResponse.js +6 -0
- package/dist/internal/api/models/ScorerExistsResponse.js.map +1 -0
- package/dist/internal/api/models/ScoringResult.d.ts +17 -0
- package/dist/internal/api/models/ScoringResult.d.ts.map +1 -0
- package/dist/internal/api/models/ScoringResult.js +6 -0
- package/dist/internal/api/models/ScoringResult.js.map +1 -0
- package/dist/internal/api/models/TraceEvaluationRun.d.ts +18 -0
- package/dist/internal/api/models/TraceEvaluationRun.d.ts.map +1 -0
- package/dist/internal/api/models/TraceEvaluationRun.js +6 -0
- package/dist/internal/api/models/TraceEvaluationRun.js.map +1 -0
- package/dist/internal/api/models.d.ts +24 -0
- package/dist/internal/api/models.d.ts.map +1 -0
- package/dist/internal/api/models.js +24 -0
- package/dist/internal/api/models.js.map +1 -0
- package/dist/scorers/api-scorer.d.ts +23 -0
- package/dist/scorers/api-scorer.d.ts.map +1 -0
- package/dist/scorers/api-scorer.js +36 -0
- package/dist/scorers/api-scorer.js.map +1 -0
- package/dist/scorers/api_scorers/answer-correctness-scorer.d.ts +11 -0
- package/dist/scorers/api_scorers/answer-correctness-scorer.d.ts.map +1 -0
- package/dist/scorers/api_scorers/answer-correctness-scorer.js +20 -0
- package/dist/scorers/api_scorers/answer-correctness-scorer.js.map +1 -0
- package/dist/scorers/api_scorers/answer-relevancy-scorer.d.ts +11 -0
- package/dist/scorers/api_scorers/answer-relevancy-scorer.d.ts.map +1 -0
- package/dist/scorers/api_scorers/answer-relevancy-scorer.js +19 -0
- package/dist/scorers/api_scorers/answer-relevancy-scorer.js.map +1 -0
- package/dist/scorers/api_scorers/faithfulness-scorer.d.ts +11 -0
- package/dist/scorers/api_scorers/faithfulness-scorer.d.ts.map +1 -0
- package/dist/scorers/api_scorers/faithfulness-scorer.js +20 -0
- package/dist/scorers/api_scorers/faithfulness-scorer.js.map +1 -0
- package/dist/scorers/api_scorers/index.d.ts +2 -0
- package/dist/scorers/api_scorers/index.d.ts.map +1 -0
- package/dist/scorers/api_scorers/index.js +2 -0
- package/dist/scorers/api_scorers/index.js.map +1 -0
- package/dist/scorers/api_scorers/instruction-adherence-scorer.d.ts +11 -0
- package/dist/scorers/api_scorers/instruction-adherence-scorer.d.ts.map +1 -0
- package/dist/scorers/api_scorers/instruction-adherence-scorer.js +20 -0
- package/dist/scorers/api_scorers/instruction-adherence-scorer.js.map +1 -0
- package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.d.ts +48 -0
- package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.d.ts.map +1 -0
- package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.js +133 -0
- package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.js.map +1 -0
- package/dist/scorers/api_scorers/prompt_scorer/index.d.ts +5 -0
- package/dist/scorers/api_scorers/prompt_scorer/index.d.ts.map +1 -0
- package/dist/scorers/api_scorers/prompt_scorer/index.js +5 -0
- package/dist/scorers/api_scorers/prompt_scorer/index.js.map +1 -0
- package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.d.ts +14 -0
- package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.d.ts.map +1 -0
- package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.js +44 -0
- package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.js.map +1 -0
- package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.d.ts +7 -0
- package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.d.ts.map +1 -0
- package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.js +9 -0
- package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.js.map +1 -0
- package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.d.ts +6 -0
- package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.d.ts.map +1 -0
- package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.js +8 -0
- package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.js.map +1 -0
- package/dist/scorers/base-scorer.d.ts +10 -0
- package/dist/scorers/base-scorer.d.ts.map +1 -0
- package/dist/scorers/base-scorer.js +41 -0
- package/dist/scorers/base-scorer.js.map +1 -0
- package/dist/scorers/index.d.ts +4 -0
- package/dist/scorers/index.d.ts.map +1 -0
- package/dist/scorers/index.js +4 -0
- package/dist/scorers/index.js.map +1 -0
- package/dist/tracer/OpenTelemetryKeys.d.ts +12 -0
- package/dist/tracer/OpenTelemetryKeys.d.ts.map +1 -0
- package/dist/tracer/OpenTelemetryKeys.js +12 -0
- package/dist/tracer/OpenTelemetryKeys.js.map +1 -0
- package/dist/tracer/Tracer.d.ts +47 -0
- package/dist/tracer/Tracer.d.ts.map +1 -0
- package/dist/tracer/Tracer.js +235 -0
- package/dist/tracer/Tracer.js.map +1 -0
- package/dist/tracer/TracerConfiguration.d.ts +71 -0
- package/dist/tracer/TracerConfiguration.d.ts.map +1 -0
- package/dist/tracer/TracerConfiguration.js +111 -0
- package/dist/tracer/TracerConfiguration.js.map +1 -0
- package/dist/tracer/exporters/JudgmentSpanExporter.d.ts +36 -0
- package/dist/tracer/exporters/JudgmentSpanExporter.d.ts.map +1 -0
- package/dist/tracer/exporters/JudgmentSpanExporter.js +72 -0
- package/dist/tracer/exporters/JudgmentSpanExporter.js.map +1 -0
- package/dist/tracer/exporters/NoOpSpanExporter.d.ts +12 -0
- package/dist/tracer/exporters/NoOpSpanExporter.d.ts.map +1 -0
- package/dist/tracer/exporters/NoOpSpanExporter.js +16 -0
- package/dist/tracer/exporters/NoOpSpanExporter.js.map +1 -0
- package/dist/tracer/exporters/index.d.ts +3 -0
- package/dist/tracer/exporters/index.d.ts.map +1 -0
- package/dist/tracer/exporters/index.js +3 -0
- package/dist/tracer/exporters/index.js.map +1 -0
- package/dist/tracer/index.d.ts +5 -0
- package/dist/tracer/index.d.ts.map +1 -0
- package/dist/tracer/index.js +5 -0
- package/dist/tracer/index.js.map +1 -0
- package/dist/umd.d.ts +2 -0
- package/dist/umd.d.ts.map +1 -0
- package/dist/umd.js +2 -0
- package/dist/umd.js.map +1 -0
- package/dist/utils/annotate.d.ts +2 -0
- package/dist/utils/annotate.d.ts.map +1 -0
- package/dist/utils/annotate.js +27 -0
- package/dist/utils/annotate.js.map +1 -0
- package/dist/utils/index.d.ts +5 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +5 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/logger.d.ts +27 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +70 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/types.d.ts +4 -0
- package/dist/utils/types.d.ts.map +1 -0
- package/dist/utils/types.js +2 -0
- package/dist/utils/types.js.map +1 -0
- package/dist/utils/validation.d.ts +11 -0
- package/dist/utils/validation.d.ts.map +1 -0
- package/dist/utils/validation.js +16 -0
- package/dist/utils/validation.js.map +1 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +2 -0
- package/dist/version.js.map +1 -0
- package/package.json +57 -108
- package/LICENSE.md +0 -202
- package/README.md +0 -372
- package/dist/cjs/clients.js +0 -78
- package/dist/cjs/clients.js.map +0 -1
- package/dist/cjs/common/integrations/langgraph.js +0 -471
- package/dist/cjs/common/integrations/langgraph.js.map +0 -1
- package/dist/cjs/common/logger-instance.js +0 -64
- package/dist/cjs/common/logger-instance.js.map +0 -1
- package/dist/cjs/common/logger.js +0 -225
- package/dist/cjs/common/logger.js.map +0 -1
- package/dist/cjs/common/token-costs.js +0 -95
- package/dist/cjs/common/token-costs.js.map +0 -1
- package/dist/cjs/common/tracer.js +0 -1099
- package/dist/cjs/common/tracer.js.map +0 -1
- package/dist/cjs/constants.js +0 -347
- package/dist/cjs/constants.js.map +0 -1
- package/dist/cjs/data/datasets/eval-dataset-client.js +0 -394
- package/dist/cjs/data/datasets/eval-dataset-client.js.map +0 -1
- package/dist/cjs/data/datasets/eval-dataset.js +0 -405
- package/dist/cjs/data/datasets/eval-dataset.js.map +0 -1
- package/dist/cjs/data/example.js +0 -159
- package/dist/cjs/data/example.js.map +0 -1
- package/dist/cjs/data/result.js +0 -83
- package/dist/cjs/data/result.js.map +0 -1
- package/dist/cjs/e2etests/eval-operations.test.js +0 -282
- package/dist/cjs/e2etests/eval-operations.test.js.map +0 -1
- package/dist/cjs/e2etests/judgee-traces.test.js +0 -278
- package/dist/cjs/e2etests/judgee-traces.test.js.map +0 -1
- package/dist/cjs/evaluation-run.js +0 -136
- package/dist/cjs/evaluation-run.js.map +0 -1
- package/dist/cjs/exporters/otel-exporter.js +0 -352
- package/dist/cjs/exporters/otel-exporter.js.map +0 -1
- package/dist/cjs/index.js +0 -78
- package/dist/cjs/index.js.map +0 -1
- package/dist/cjs/judges/index.js +0 -237
- package/dist/cjs/judges/index.js.map +0 -1
- package/dist/cjs/judgment-client.js +0 -798
- package/dist/cjs/judgment-client.js.map +0 -1
- package/dist/cjs/rules.js +0 -322
- package/dist/cjs/rules.js.map +0 -1
- package/dist/cjs/run-evaluation.js +0 -700
- package/dist/cjs/run-evaluation.js.map +0 -1
- package/dist/cjs/scorers/api-scorer.js +0 -304
- package/dist/cjs/scorers/api-scorer.js.map +0 -1
- package/dist/cjs/scorers/base-scorer.js +0 -293
- package/dist/cjs/scorers/base-scorer.js.map +0 -1
- package/dist/cjs/scorers/metrics/answer-correctness/answer-correctness.js +0 -562
- package/dist/cjs/scorers/metrics/answer-correctness/answer-correctness.js.map +0 -1
- package/dist/cjs/scorers/metrics/answer-correctness/index.js +0 -19
- package/dist/cjs/scorers/metrics/answer-correctness/index.js.map +0 -1
- package/dist/cjs/scorers/metrics/answer-correctness/prompts.js +0 -175
- package/dist/cjs/scorers/metrics/answer-correctness/prompts.js.map +0 -1
- package/dist/cjs/scorers/metrics/answer-relevancy/answer-relevancy.js +0 -514
- package/dist/cjs/scorers/metrics/answer-relevancy/answer-relevancy.js.map +0 -1
- package/dist/cjs/scorers/metrics/answer-relevancy/index.js +0 -19
- package/dist/cjs/scorers/metrics/answer-relevancy/index.js.map +0 -1
- package/dist/cjs/scorers/metrics/answer-relevancy/prompts.js +0 -179
- package/dist/cjs/scorers/metrics/answer-relevancy/prompts.js.map +0 -1
- package/dist/cjs/scorers/metrics/faithfulness/faithfulness.js +0 -512
- package/dist/cjs/scorers/metrics/faithfulness/faithfulness.js.map +0 -1
- package/dist/cjs/scorers/metrics/faithfulness/index.js +0 -19
- package/dist/cjs/scorers/metrics/faithfulness/index.js.map +0 -1
- package/dist/cjs/scorers/metrics/faithfulness/prompts.js +0 -232
- package/dist/cjs/scorers/metrics/faithfulness/prompts.js.map +0 -1
- package/dist/cjs/scorers/metrics/hallucination/hallucination.js +0 -386
- package/dist/cjs/scorers/metrics/hallucination/hallucination.js.map +0 -1
- package/dist/cjs/scorers/metrics/hallucination/index.js +0 -11
- package/dist/cjs/scorers/metrics/hallucination/index.js.map +0 -1
- package/dist/cjs/scorers/metrics/hallucination/prompts.js +0 -106
- package/dist/cjs/scorers/metrics/hallucination/prompts.js.map +0 -1
- package/dist/cjs/scorers/metrics/instruction-adherence/index.js +0 -19
- package/dist/cjs/scorers/metrics/instruction-adherence/index.js.map +0 -1
- package/dist/cjs/scorers/metrics/instruction-adherence/instruction-adherence.js +0 -378
- package/dist/cjs/scorers/metrics/instruction-adherence/instruction-adherence.js.map +0 -1
- package/dist/cjs/scorers/metrics/instruction-adherence/prompts.js +0 -124
- package/dist/cjs/scorers/metrics/instruction-adherence/prompts.js.map +0 -1
- package/dist/esm/clients.js +0 -47
- package/dist/esm/clients.js.map +0 -1
- package/dist/esm/common/integrations/langgraph.js +0 -444
- package/dist/esm/common/integrations/langgraph.js.map +0 -1
- package/dist/esm/common/logger-instance.js +0 -59
- package/dist/esm/common/logger-instance.js.map +0 -1
- package/dist/esm/common/logger.js +0 -208
- package/dist/esm/common/logger.js.map +0 -1
- package/dist/esm/common/token-costs.js +0 -91
- package/dist/esm/common/token-costs.js.map +0 -1
- package/dist/esm/common/tracer.js +0 -1093
- package/dist/esm/common/tracer.js.map +0 -1
- package/dist/esm/constants.js +0 -341
- package/dist/esm/constants.js.map +0 -1
- package/dist/esm/data/datasets/eval-dataset-client.js +0 -387
- package/dist/esm/data/datasets/eval-dataset-client.js.map +0 -1
- package/dist/esm/data/datasets/eval-dataset.js +0 -375
- package/dist/esm/data/datasets/eval-dataset.js.map +0 -1
- package/dist/esm/data/example.js +0 -154
- package/dist/esm/data/example.js.map +0 -1
- package/dist/esm/data/result.js +0 -78
- package/dist/esm/data/result.js.map +0 -1
- package/dist/esm/e2etests/eval-operations.test.js +0 -254
- package/dist/esm/e2etests/eval-operations.test.js.map +0 -1
- package/dist/esm/e2etests/judgee-traces.test.js +0 -253
- package/dist/esm/e2etests/judgee-traces.test.js.map +0 -1
- package/dist/esm/evaluation-run.js +0 -132
- package/dist/esm/evaluation-run.js.map +0 -1
- package/dist/esm/exporters/otel-exporter.js +0 -348
- package/dist/esm/exporters/otel-exporter.js.map +0 -1
- package/dist/esm/index.js +0 -20
- package/dist/esm/index.js.map +0 -1
- package/dist/esm/judges/index.js +0 -205
- package/dist/esm/judges/index.js.map +0 -1
- package/dist/esm/judgment-client.js +0 -768
- package/dist/esm/judgment-client.js.map +0 -1
- package/dist/esm/rules.js +0 -314
- package/dist/esm/rules.js.map +0 -1
- package/dist/esm/run-evaluation.js +0 -681
- package/dist/esm/run-evaluation.js.map +0 -1
- package/dist/esm/scorers/api-scorer.js +0 -286
- package/dist/esm/scorers/api-scorer.js.map +0 -1
- package/dist/esm/scorers/base-scorer.js +0 -287
- package/dist/esm/scorers/base-scorer.js.map +0 -1
- package/dist/esm/scorers/metrics/answer-correctness/answer-correctness.js +0 -558
- package/dist/esm/scorers/metrics/answer-correctness/answer-correctness.js.map +0 -1
- package/dist/esm/scorers/metrics/answer-correctness/index.js +0 -3
- package/dist/esm/scorers/metrics/answer-correctness/index.js.map +0 -1
- package/dist/esm/scorers/metrics/answer-correctness/prompts.js +0 -171
- package/dist/esm/scorers/metrics/answer-correctness/prompts.js.map +0 -1
- package/dist/esm/scorers/metrics/answer-relevancy/answer-relevancy.js +0 -510
- package/dist/esm/scorers/metrics/answer-relevancy/answer-relevancy.js.map +0 -1
- package/dist/esm/scorers/metrics/answer-relevancy/index.js +0 -3
- package/dist/esm/scorers/metrics/answer-relevancy/index.js.map +0 -1
- package/dist/esm/scorers/metrics/answer-relevancy/prompts.js +0 -175
- package/dist/esm/scorers/metrics/answer-relevancy/prompts.js.map +0 -1
- package/dist/esm/scorers/metrics/faithfulness/faithfulness.js +0 -508
- package/dist/esm/scorers/metrics/faithfulness/faithfulness.js.map +0 -1
- package/dist/esm/scorers/metrics/faithfulness/index.js +0 -3
- package/dist/esm/scorers/metrics/faithfulness/index.js.map +0 -1
- package/dist/esm/scorers/metrics/faithfulness/prompts.js +0 -228
- package/dist/esm/scorers/metrics/faithfulness/prompts.js.map +0 -1
- package/dist/esm/scorers/metrics/hallucination/hallucination.js +0 -382
- package/dist/esm/scorers/metrics/hallucination/hallucination.js.map +0 -1
- package/dist/esm/scorers/metrics/hallucination/index.js +0 -3
- package/dist/esm/scorers/metrics/hallucination/index.js.map +0 -1
- package/dist/esm/scorers/metrics/hallucination/prompts.js +0 -102
- package/dist/esm/scorers/metrics/hallucination/prompts.js.map +0 -1
- package/dist/esm/scorers/metrics/instruction-adherence/index.js +0 -3
- package/dist/esm/scorers/metrics/instruction-adherence/index.js.map +0 -1
- package/dist/esm/scorers/metrics/instruction-adherence/instruction-adherence.js +0 -374
- package/dist/esm/scorers/metrics/instruction-adherence/instruction-adherence.js.map +0 -1
- package/dist/esm/scorers/metrics/instruction-adherence/prompts.js +0 -120
- package/dist/esm/scorers/metrics/instruction-adherence/prompts.js.map +0 -1
- package/dist/types/clients.d.ts +0 -6
- package/dist/types/common/integrations/langgraph.d.ts +0 -39
- package/dist/types/common/logger-instance.d.ts +0 -3
- package/dist/types/common/logger.d.ts +0 -54
- package/dist/types/common/token-costs.d.ts +0 -36
- package/dist/types/common/tracer.d.ts +0 -249
- package/dist/types/constants.d.ts +0 -51
- package/dist/types/data/datasets/eval-dataset-client.d.ts +0 -44
- package/dist/types/data/datasets/eval-dataset.d.ts +0 -45
- package/dist/types/data/example.d.ts +0 -82
- package/dist/types/data/result.d.ts +0 -51
- package/dist/types/e2etests/eval-operations.test.d.ts +0 -5
- package/dist/types/e2etests/judgee-traces.test.d.ts +0 -5
- package/dist/types/evaluation-run.d.ts +0 -44
- package/dist/types/exporters/otel-exporter.d.ts +0 -16
- package/dist/types/index.d.ts +0 -11
- package/dist/types/judges/index.d.ts +0 -50
- package/dist/types/judgment-client.d.ts +0 -149
- package/dist/types/rules.d.ts +0 -120
- package/dist/types/run-evaluation.d.ts +0 -78
- package/dist/types/scorers/api-scorer.d.ts +0 -84
- package/dist/types/scorers/base-scorer.d.ts +0 -162
- package/dist/types/scorers/metrics/answer-correctness/answer-correctness.d.ts +0 -80
- package/dist/types/scorers/metrics/answer-correctness/index.d.ts +0 -2
- package/dist/types/scorers/metrics/answer-correctness/prompts.d.ts +0 -71
- package/dist/types/scorers/metrics/answer-relevancy/answer-relevancy.d.ts +0 -74
- package/dist/types/scorers/metrics/answer-relevancy/index.d.ts +0 -2
- package/dist/types/scorers/metrics/answer-relevancy/prompts.d.ts +0 -71
- package/dist/types/scorers/metrics/faithfulness/faithfulness.d.ts +0 -73
- package/dist/types/scorers/metrics/faithfulness/index.d.ts +0 -2
- package/dist/types/scorers/metrics/faithfulness/prompts.d.ts +0 -94
- package/dist/types/scorers/metrics/hallucination/hallucination.d.ts +0 -63
- package/dist/types/scorers/metrics/hallucination/index.d.ts +0 -3
- package/dist/types/scorers/metrics/hallucination/prompts.d.ts +0 -63
- package/dist/types/scorers/metrics/instruction-adherence/index.d.ts +0 -2
- package/dist/types/scorers/metrics/instruction-adherence/instruction-adherence.d.ts +0 -63
- package/dist/types/scorers/metrics/instruction-adherence/prompts.d.ts +0 -78
|
@@ -1,1099 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.TraceManagerClient = exports.TraceClient = exports.Tracer = void 0;
|
|
16
|
-
exports.wrap = wrap;
|
|
17
|
-
// Core Node.js imports
|
|
18
|
-
const uuid_1 = require("uuid");
|
|
19
|
-
const async_hooks_1 = require("async_hooks");
|
|
20
|
-
// Installed SDKs
|
|
21
|
-
const openai_1 = __importDefault(require("openai"));
|
|
22
|
-
const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
|
|
23
|
-
// Local Imports
|
|
24
|
-
const constants_js_1 = require("../constants.js");
|
|
25
|
-
const base_scorer_js_1 = require("../scorers/base-scorer.js");
|
|
26
|
-
const logger_instance_js_1 = __importDefault(require("./logger-instance.js")); // Use the shared winston logger instance
|
|
27
|
-
// On their own, span() and trace() are fully synchronous
|
|
28
|
-
// The issue is with asynchronous functions
|
|
29
|
-
// To address this, I made a wrapper for async functions that'll handle all the context stuff
|
|
30
|
-
class TraceClientContext {
|
|
31
|
-
constructor() {
|
|
32
|
-
this.entries = [];
|
|
33
|
-
this.entryStack = [];
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
const traceClientContextAsyncLocalStorage = new async_hooks_1.AsyncLocalStorage();
|
|
37
|
-
// We could use .enterWith(), but the documentation advises against it
|
|
38
|
-
let rootTraceClientContext = new TraceClientContext();
|
|
39
|
-
function getTraceClientContext() {
|
|
40
|
-
var _a;
|
|
41
|
-
return (_a = traceClientContextAsyncLocalStorage.getStore()) !== null && _a !== void 0 ? _a : rootTraceClientContext;
|
|
42
|
-
}
|
|
43
|
-
function asyncFunctionWrapper(func) {
|
|
44
|
-
return (...args) => __awaiter(this, void 0, void 0, function* () {
|
|
45
|
-
const parentTraceClientContext = getTraceClientContext();
|
|
46
|
-
const traceClientContext = new TraceClientContext();
|
|
47
|
-
const lastEntry = parentTraceClientContext.entryStack.at(-1);
|
|
48
|
-
if (lastEntry) {
|
|
49
|
-
traceClientContext.entryStack.push(lastEntry);
|
|
50
|
-
}
|
|
51
|
-
const result = yield traceClientContextAsyncLocalStorage
|
|
52
|
-
.run(traceClientContext, () => func(...args));
|
|
53
|
-
if (traceClientContext.entries.length > parentTraceClientContext.entries.length) {
|
|
54
|
-
[traceClientContext.entries, parentTraceClientContext.entries] =
|
|
55
|
-
[parentTraceClientContext.entries, traceClientContext.entries];
|
|
56
|
-
}
|
|
57
|
-
parentTraceClientContext.entries.push(...traceClientContext.entries);
|
|
58
|
-
return result;
|
|
59
|
-
});
|
|
60
|
-
}
|
|
61
|
-
// --- API Interaction Client ---
|
|
62
|
-
/**
|
|
63
|
-
* Client for interacting with Judgment trace API endpoints.
|
|
64
|
-
*/
|
|
65
|
-
class TraceManagerClient {
|
|
66
|
-
constructor(apiKey, organizationId) {
|
|
67
|
-
if (!apiKey) {
|
|
68
|
-
throw new Error("TraceManagerClient requires a Judgment API key.");
|
|
69
|
-
}
|
|
70
|
-
if (!organizationId) {
|
|
71
|
-
throw new Error("TraceManagerClient requires a Judgment Organization ID.");
|
|
72
|
-
}
|
|
73
|
-
this.apiKey = apiKey;
|
|
74
|
-
this.organizationId = organizationId;
|
|
75
|
-
}
|
|
76
|
-
_fetch(url_1) {
|
|
77
|
-
return __awaiter(this, arguments, void 0, function* (url, options = {}) {
|
|
78
|
-
const headers = Object.assign({ 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.apiKey}`, 'X-Organization-Id': this.organizationId }, (options.headers || {}));
|
|
79
|
-
try {
|
|
80
|
-
// Use isomorphic fetch (available globally in modern Node.js and browsers)
|
|
81
|
-
const response = yield fetch(url, Object.assign(Object.assign({}, options), { headers: headers }));
|
|
82
|
-
// We will return the response object even if !response.ok
|
|
83
|
-
// The caller (e.g., saveTrace) is responsible for checking response.ok or response.status
|
|
84
|
-
// Handle cases where the response might be empty (e.g., 204 No Content on DELETE)
|
|
85
|
-
if (response.status === 204) {
|
|
86
|
-
return null; // Indicate success with no content
|
|
87
|
-
}
|
|
88
|
-
return response;
|
|
89
|
-
}
|
|
90
|
-
catch (error) {
|
|
91
|
-
console.error(`Network or fetch error during ${options.method || 'GET'} ${url}:`, error);
|
|
92
|
-
// Re-throw or handle as appropriate for the application context
|
|
93
|
-
throw error;
|
|
94
|
-
}
|
|
95
|
-
});
|
|
96
|
-
}
|
|
97
|
-
fetchTrace(traceId) {
|
|
98
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
99
|
-
return this._fetch(constants_js_1.JUDGMENT_TRACES_FETCH_API_URL, {
|
|
100
|
-
method: 'POST',
|
|
101
|
-
body: JSON.stringify({ trace_id: traceId }),
|
|
102
|
-
});
|
|
103
|
-
});
|
|
104
|
-
}
|
|
105
|
-
saveTrace(traceData) {
|
|
106
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
107
|
-
// _fetch now returns the raw response object or throws on network error
|
|
108
|
-
const response = yield this._fetch(constants_js_1.JUDGMENT_TRACES_SAVE_API_URL, {
|
|
109
|
-
method: 'POST',
|
|
110
|
-
body: JSON.stringify(traceData), // Stringify directly here again
|
|
111
|
-
});
|
|
112
|
-
// Check if _fetch threw a network error (caught below) or returned an invalid object
|
|
113
|
-
if (!response) {
|
|
114
|
-
// This case should ideally be caught by _fetch's catch block, but double-check
|
|
115
|
-
throw new Error('Failed to save trace data: No response received from API.');
|
|
116
|
-
}
|
|
117
|
-
// Now, check the status code on the received response object
|
|
118
|
-
if (response.status === 400) {
|
|
119
|
-
// Attempt to get error body for more info
|
|
120
|
-
const errorBody = yield response.text();
|
|
121
|
-
throw new Error(`Failed to save trace data: Check your Trace name for conflicts, set overwrite=True to overwrite existing traces: ${response.status} ${response.statusText || ''} - ${errorBody}`);
|
|
122
|
-
}
|
|
123
|
-
else if (!response.ok) { // Handles other errors (5xx, 4xx except 400)
|
|
124
|
-
const errorBody = yield response.text();
|
|
125
|
-
throw new Error(`Failed to save trace data: Status ${response.status} ${response.statusText || '(No status text)'} - ${errorBody}`);
|
|
126
|
-
}
|
|
127
|
-
// --- Success Path ---
|
|
128
|
-
// Optionally log the UI URL (needs JSON parsing)
|
|
129
|
-
let responseData = null;
|
|
130
|
-
try {
|
|
131
|
-
// Handle 204 No Content specifically
|
|
132
|
-
if (response.status === 204) {
|
|
133
|
-
responseData = null; // Or maybe { success: true }?
|
|
134
|
-
}
|
|
135
|
-
else {
|
|
136
|
-
responseData = yield response.json(); // Parse JSON only on success
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
catch (parseError) {
|
|
140
|
-
logger_instance_js_1.default.warn("Failed to parse successful API response JSON.", { error: parseError });
|
|
141
|
-
// Depending on requirements, maybe throw, maybe return a default success object
|
|
142
|
-
throw new Error(`API request succeeded (${response.status}), but failed to parse JSON response.`);
|
|
143
|
-
}
|
|
144
|
-
if (responseData === null || responseData === void 0 ? void 0 : responseData.ui_results_url) {
|
|
145
|
-
console.info(`
|
|
146
|
-
🔍 View trace: ${responseData.ui_results_url}
|
|
147
|
-
`);
|
|
148
|
-
}
|
|
149
|
-
// Return the parsed data (or null for 204)
|
|
150
|
-
return responseData;
|
|
151
|
-
});
|
|
152
|
-
}
|
|
153
|
-
deleteTrace(traceId) {
|
|
154
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
155
|
-
// Assuming DELETE method is correct based on REST principles for the delete endpoint
|
|
156
|
-
return this._fetch(constants_js_1.JUDGMENT_TRACES_DELETE_API_URL, {
|
|
157
|
-
method: 'DELETE',
|
|
158
|
-
body: JSON.stringify({ trace_ids: [traceId] }),
|
|
159
|
-
});
|
|
160
|
-
});
|
|
161
|
-
}
|
|
162
|
-
deleteTraces(traceIds) {
|
|
163
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
164
|
-
return this._fetch(constants_js_1.JUDGMENT_TRACES_DELETE_API_URL, {
|
|
165
|
-
method: 'DELETE',
|
|
166
|
-
body: JSON.stringify({ trace_ids: traceIds }),
|
|
167
|
-
});
|
|
168
|
-
});
|
|
169
|
-
}
|
|
170
|
-
/**
|
|
171
|
-
* Calculate token costs directly using the API endpoint.
|
|
172
|
-
* This is more accurate than client-side calculation as it uses the most up-to-date pricing.
|
|
173
|
-
*
|
|
174
|
-
* @param model The model name (e.g. 'gpt-4', 'claude-3-opus-20240229')
|
|
175
|
-
* @param promptTokens Number of tokens in the prompt/input
|
|
176
|
-
* @param completionTokens Number of tokens in the completion/output
|
|
177
|
-
* @returns Object containing token counts and calculated costs in USD
|
|
178
|
-
*/
|
|
179
|
-
calculateTokenCosts(model, promptTokens, completionTokens) {
|
|
180
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
181
|
-
try {
|
|
182
|
-
// Use the new calculation endpoint
|
|
183
|
-
const response = yield this._fetch(constants_js_1.JUDGMENT_CALCULATE_TOKEN_COSTS_API_URL, {
|
|
184
|
-
method: 'POST',
|
|
185
|
-
body: JSON.stringify({
|
|
186
|
-
model,
|
|
187
|
-
prompt_tokens: promptTokens,
|
|
188
|
-
completion_tokens: completionTokens
|
|
189
|
-
})
|
|
190
|
-
});
|
|
191
|
-
// Check if the response is okay and parse JSON
|
|
192
|
-
if (response && response.ok) {
|
|
193
|
-
const data = yield response.json();
|
|
194
|
-
return data;
|
|
195
|
-
}
|
|
196
|
-
else if (response) {
|
|
197
|
-
// Log error if response was not ok
|
|
198
|
-
const errorBody = yield response.text();
|
|
199
|
-
logger_instance_js_1.default.warn(`API error calculating token costs for model ${model}: ${response.status} ${response.statusText}`, { errorBody });
|
|
200
|
-
return null;
|
|
201
|
-
}
|
|
202
|
-
else {
|
|
203
|
-
// Handle cases where _fetch might return null or undefined (though it shouldn't with current implementation)
|
|
204
|
-
logger_instance_js_1.default.warn(`No response received when calculating token costs for model ${model}.`);
|
|
205
|
-
return null;
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
catch (error) {
|
|
209
|
-
logger_instance_js_1.default.warn(`Failed to calculate token costs for model ${model}.`, { error: error instanceof Error ? error.message : String(error) });
|
|
210
|
-
return null;
|
|
211
|
-
}
|
|
212
|
-
});
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
exports.TraceManagerClient = TraceManagerClient;
|
|
216
|
-
// --- Helper Functions ---
|
|
217
|
-
// Helper function to sanitize names (e.g., replace spaces with underscores)
|
|
218
|
-
function sanitizeName(name) {
|
|
219
|
-
// Replace spaces with underscores and remove potentially problematic characters
|
|
220
|
-
// You can adjust the regex further if other characters cause issues.
|
|
221
|
-
return name.replace(/\s+/g, '_').replace(/[^a-zA-Z0-9_.-]/g, '');
|
|
222
|
-
}
|
|
223
|
-
// --- Core Trace Classes ---
|
|
224
|
-
/**
|
|
225
|
-
* Represents an ongoing trace context.
|
|
226
|
-
*/
|
|
227
|
-
class TraceClient {
|
|
228
|
-
constructor(config) {
|
|
229
|
-
var _a, _b, _c, _d, _e;
|
|
230
|
-
this.traceManager = null; // Made public for wrap access
|
|
231
|
-
this._spanDepths = {}; // Track depth of active spans
|
|
232
|
-
this.pendingEvaluationRuns = []; // <-- ADDED: Store pending evaluations
|
|
233
|
-
this.traceId = config.traceId || (0, uuid_1.v4)();
|
|
234
|
-
this.originalName = config.name || 'default_trace'; // Store original
|
|
235
|
-
this.name = sanitizeName(this.originalName); // Use sanitized name internally
|
|
236
|
-
// If the sanitized name is empty, fallback to a default
|
|
237
|
-
if (!this.name) {
|
|
238
|
-
console.warn(`Original trace name "${this.originalName}" sanitized to empty string. Using default_trace_${this.traceId.substring(0, 8)}.`);
|
|
239
|
-
this.name = `default_trace_${this.traceId.substring(0, 8)}`;
|
|
240
|
-
}
|
|
241
|
-
this.projectName = (_a = config.projectName) !== null && _a !== void 0 ? _a : config.tracer.projectName;
|
|
242
|
-
this.overwrite = (_b = config.overwrite) !== null && _b !== void 0 ? _b : false;
|
|
243
|
-
this.rules = (_c = config.rules) !== null && _c !== void 0 ? _c : [];
|
|
244
|
-
// Determine effective monitoring status based on tracer and API keys
|
|
245
|
-
let effectiveMonitoring = (_d = config.enableMonitoring) !== null && _d !== void 0 ? _d : config.tracer.enableMonitoring;
|
|
246
|
-
if (effectiveMonitoring && (!config.apiKey || !config.organizationId)) {
|
|
247
|
-
console.warn(`TraceClient ${this.traceId}: Monitoring requires JUDGMENT_API_KEY and JUDGMENT_ORG_ID. Disabling monitoring for this trace.`);
|
|
248
|
-
effectiveMonitoring = false;
|
|
249
|
-
}
|
|
250
|
-
this.enableMonitoring = effectiveMonitoring;
|
|
251
|
-
// Evaluations depend on monitoring
|
|
252
|
-
this.enableEvaluations = effectiveMonitoring && ((_e = config.enableEvaluations) !== null && _e !== void 0 ? _e : config.tracer.enableEvaluations);
|
|
253
|
-
this.parentTraceId = config.parentTraceId;
|
|
254
|
-
this.parentName = config.parentName;
|
|
255
|
-
this.apiKey = config.apiKey;
|
|
256
|
-
this.organizationId = config.organizationId;
|
|
257
|
-
this.startTime = Date.now() / 1000;
|
|
258
|
-
if (this.enableMonitoring) {
|
|
259
|
-
this.traceManager = new TraceManagerClient(this.apiKey, this.organizationId);
|
|
260
|
-
}
|
|
261
|
-
}
|
|
262
|
-
addEntry(entry) {
|
|
263
|
-
const traceClientContext = getTraceClientContext();
|
|
264
|
-
if (this.enableMonitoring) {
|
|
265
|
-
traceClientContext.entries.push(entry);
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
recordInput(inputs) {
|
|
269
|
-
const traceClientContext = getTraceClientContext();
|
|
270
|
-
const currentEntry = traceClientContext.entryStack.at(-1);
|
|
271
|
-
if (!currentEntry || !currentEntry.span_id) {
|
|
272
|
-
console.warn(`No current entry to record input to\nStack trace: ${new Error().stack}`);
|
|
273
|
-
return;
|
|
274
|
-
}
|
|
275
|
-
this.addEntry({
|
|
276
|
-
type: 'input',
|
|
277
|
-
span_id: currentEntry.span_id,
|
|
278
|
-
inputs,
|
|
279
|
-
function: currentEntry.function,
|
|
280
|
-
depth: this._spanDepths[currentEntry.span_id],
|
|
281
|
-
created_at: Date.now() / 1000,
|
|
282
|
-
span_type: currentEntry.span_type,
|
|
283
|
-
message: `Inputs to ${currentEntry.function}`
|
|
284
|
-
});
|
|
285
|
-
}
|
|
286
|
-
recordOutput(output) {
|
|
287
|
-
const traceClientContext = getTraceClientContext();
|
|
288
|
-
const currentEntry = traceClientContext.entryStack.at(-1);
|
|
289
|
-
if (!currentEntry || !currentEntry.span_id) {
|
|
290
|
-
console.warn(`No current entry to record output to\nStack trace: ${new Error().stack}`);
|
|
291
|
-
return;
|
|
292
|
-
}
|
|
293
|
-
this.addEntry({
|
|
294
|
-
type: 'output',
|
|
295
|
-
span_id: currentEntry.span_id,
|
|
296
|
-
output,
|
|
297
|
-
function: currentEntry.function,
|
|
298
|
-
depth: this._spanDepths[currentEntry.span_id],
|
|
299
|
-
created_at: Date.now() / 1000,
|
|
300
|
-
span_type: currentEntry.span_type,
|
|
301
|
-
message: `Output from ${currentEntry.function}`
|
|
302
|
-
});
|
|
303
|
-
}
|
|
304
|
-
recordError(error) {
|
|
305
|
-
const traceClientContext = getTraceClientContext();
|
|
306
|
-
const currentEntry = traceClientContext.entryStack.at(-1);
|
|
307
|
-
if (!currentEntry || !currentEntry.span_id) {
|
|
308
|
-
console.warn(`No current entry to record error to\nStack trace: ${new Error().stack}`);
|
|
309
|
-
return;
|
|
310
|
-
}
|
|
311
|
-
this.addEntry({
|
|
312
|
-
type: 'error',
|
|
313
|
-
span_id: currentEntry.span_id,
|
|
314
|
-
output: error,
|
|
315
|
-
function: currentEntry.function,
|
|
316
|
-
depth: this._spanDepths[currentEntry.span_id],
|
|
317
|
-
created_at: Date.now() / 1000,
|
|
318
|
-
span_type: currentEntry.span_type,
|
|
319
|
-
message: `Error from ${currentEntry.function}`
|
|
320
|
-
});
|
|
321
|
-
}
|
|
322
|
-
startSpan(name, options = {}) {
|
|
323
|
-
var _a;
|
|
324
|
-
const traceClientContext = getTraceClientContext();
|
|
325
|
-
const parentEntry = traceClientContext.entryStack.at(-1);
|
|
326
|
-
const spanId = (0, uuid_1.v4)();
|
|
327
|
-
const spanType = (_a = options.spanType) !== null && _a !== void 0 ? _a : 'span';
|
|
328
|
-
const startTime = Date.now() / 1000;
|
|
329
|
-
let depth = 0, parentSpanId = undefined;
|
|
330
|
-
if (parentEntry && parentEntry.span_id) {
|
|
331
|
-
depth = this._spanDepths[parentEntry.span_id] + 1;
|
|
332
|
-
parentSpanId = parentEntry.span_id;
|
|
333
|
-
}
|
|
334
|
-
this._spanDepths[spanId] = depth;
|
|
335
|
-
const entry = {
|
|
336
|
-
type: 'enter',
|
|
337
|
-
function: name,
|
|
338
|
-
span_id: spanId,
|
|
339
|
-
depth: depth,
|
|
340
|
-
created_at: startTime,
|
|
341
|
-
span_type: spanType,
|
|
342
|
-
parent_span_id: parentSpanId,
|
|
343
|
-
message: name
|
|
344
|
-
};
|
|
345
|
-
this.addEntry(entry);
|
|
346
|
-
traceClientContext.entryStack.push(entry);
|
|
347
|
-
}
|
|
348
|
-
endSpan() {
|
|
349
|
-
const traceClientContext = getTraceClientContext();
|
|
350
|
-
const enterEntry = traceClientContext.entryStack.pop();
|
|
351
|
-
if (!enterEntry || !enterEntry.span_id) {
|
|
352
|
-
console.warn("No enter entry to end");
|
|
353
|
-
return;
|
|
354
|
-
}
|
|
355
|
-
const endTime = Date.now() / 1000;
|
|
356
|
-
const duration = endTime - enterEntry.created_at;
|
|
357
|
-
this.addEntry({
|
|
358
|
-
type: 'exit',
|
|
359
|
-
function: enterEntry.function,
|
|
360
|
-
span_id: enterEntry.span_id,
|
|
361
|
-
depth: this._spanDepths[enterEntry.span_id],
|
|
362
|
-
created_at: endTime,
|
|
363
|
-
duration: duration,
|
|
364
|
-
span_type: enterEntry.span_type,
|
|
365
|
-
message: `← ${enterEntry.function}`
|
|
366
|
-
});
|
|
367
|
-
// Clean up depth tracking
|
|
368
|
-
delete this._spanDepths[enterEntry.span_id];
|
|
369
|
-
}
|
|
370
|
-
*span(name, options = {}) {
|
|
371
|
-
if (!this.enableMonitoring) {
|
|
372
|
-
yield this;
|
|
373
|
-
}
|
|
374
|
-
else {
|
|
375
|
-
this.startSpan(name, options);
|
|
376
|
-
yield this;
|
|
377
|
-
this.endSpan();
|
|
378
|
-
}
|
|
379
|
-
}
|
|
380
|
-
/**
|
|
381
|
-
* Retrieves the ID of the currently active span in this trace context.
|
|
382
|
-
* Relies on AsyncLocalStorage context established by observe/span.
|
|
383
|
-
* @returns {string | undefined} The ID of the current span, or undefined if none is active.
|
|
384
|
-
*/
|
|
385
|
-
getCurrentSpanId() {
|
|
386
|
-
const traceClientContext = getTraceClientContext(); // Internal function using AsyncLocalStorage
|
|
387
|
-
if (!traceClientContext) {
|
|
388
|
-
// Should ideally not happen if called within an observe/trace context
|
|
389
|
-
console.warn("[Judgeval] getCurrentSpanId called outside of an active trace context.");
|
|
390
|
-
return undefined;
|
|
391
|
-
}
|
|
392
|
-
const currentEntry = traceClientContext.entryStack.at(-1); // Get the latest 'enter' entry
|
|
393
|
-
if (!currentEntry) {
|
|
394
|
-
// This might happen if called right at the start of a trace before the first span
|
|
395
|
-
// console.warn("[Judgeval] getCurrentSpanId called but span stack is empty.");
|
|
396
|
-
return undefined;
|
|
397
|
-
}
|
|
398
|
-
return currentEntry.span_id;
|
|
399
|
-
}
|
|
400
|
-
getDuration() {
|
|
401
|
-
return (Date.now() / 1000) - this.startTime;
|
|
402
|
-
}
|
|
403
|
-
condenseTrace(rawEntries) {
|
|
404
|
-
var _a, _b, _c, _d, _e;
|
|
405
|
-
const spansById = {};
|
|
406
|
-
for (const entry of rawEntries) {
|
|
407
|
-
const spanId = entry.span_id;
|
|
408
|
-
if (!spanId)
|
|
409
|
-
continue;
|
|
410
|
-
if (!spansById[spanId]) {
|
|
411
|
-
spansById[spanId] = {
|
|
412
|
-
span_id: spanId,
|
|
413
|
-
function: entry.function || 'unknown',
|
|
414
|
-
depth: (_a = entry.depth) !== null && _a !== void 0 ? _a : 0,
|
|
415
|
-
created_at: new Date(((_b = entry.created_at) !== null && _b !== void 0 ? _b : 0) * 1000).toISOString(), // Convert number to ISO string
|
|
416
|
-
trace_id: this.traceId, // Add trace_id
|
|
417
|
-
parent_span_id: entry.parent_span_id,
|
|
418
|
-
span_type: entry.span_type || 'span',
|
|
419
|
-
inputs: null,
|
|
420
|
-
output: null,
|
|
421
|
-
duration: null,
|
|
422
|
-
children: []
|
|
423
|
-
};
|
|
424
|
-
}
|
|
425
|
-
const currentSpanData = spansById[spanId];
|
|
426
|
-
switch (entry.type) {
|
|
427
|
-
case 'enter':
|
|
428
|
-
currentSpanData.function = entry.function || currentSpanData.function;
|
|
429
|
-
currentSpanData.depth = (_c = entry.depth) !== null && _c !== void 0 ? _c : currentSpanData.depth;
|
|
430
|
-
currentSpanData.created_at = new Date(((_d = entry.created_at) !== null && _d !== void 0 ? _d : 0) * 1000).toISOString(); // Ensure created_at is string on update
|
|
431
|
-
currentSpanData.parent_span_id = entry.parent_span_id;
|
|
432
|
-
currentSpanData.span_type = entry.span_type || currentSpanData.span_type;
|
|
433
|
-
currentSpanData.start_time = entry.created_at; // Keep original number for duration calc
|
|
434
|
-
break;
|
|
435
|
-
case 'exit':
|
|
436
|
-
currentSpanData.duration = (_e = entry.duration) !== null && _e !== void 0 ? _e : currentSpanData.duration;
|
|
437
|
-
currentSpanData.end_time = entry.created_at; // Keep original number for duration calc
|
|
438
|
-
if (currentSpanData.duration === null && currentSpanData.start_time && currentSpanData.end_time) {
|
|
439
|
-
currentSpanData.duration = currentSpanData.end_time - currentSpanData.start_time;
|
|
440
|
-
}
|
|
441
|
-
break;
|
|
442
|
-
case 'input':
|
|
443
|
-
if (currentSpanData.inputs === null && entry.inputs) {
|
|
444
|
-
currentSpanData.inputs = entry.inputs;
|
|
445
|
-
}
|
|
446
|
-
else if (typeof currentSpanData.inputs === 'object' && typeof entry.inputs === 'object') {
|
|
447
|
-
currentSpanData.inputs = Object.assign(Object.assign({}, currentSpanData.inputs), entry.inputs);
|
|
448
|
-
}
|
|
449
|
-
break;
|
|
450
|
-
case 'output':
|
|
451
|
-
case 'error':
|
|
452
|
-
currentSpanData.output = entry.output;
|
|
453
|
-
break;
|
|
454
|
-
}
|
|
455
|
-
}
|
|
456
|
-
const spansList = Object.values(spansById).map(span => {
|
|
457
|
-
if (span.duration === null && span.start_time && span.end_time) {
|
|
458
|
-
span.duration = span.end_time - span.start_time;
|
|
459
|
-
}
|
|
460
|
-
delete span.start_time;
|
|
461
|
-
delete span.end_time;
|
|
462
|
-
return span;
|
|
463
|
-
});
|
|
464
|
-
const childrenMap = {};
|
|
465
|
-
const roots = [];
|
|
466
|
-
const spanMap = {};
|
|
467
|
-
const sortedCondensedList = [];
|
|
468
|
-
const visited = new Set();
|
|
469
|
-
for (const span of spansList) {
|
|
470
|
-
spanMap[span.span_id] = span;
|
|
471
|
-
const parentId = span.parent_span_id;
|
|
472
|
-
if (parentId === undefined || parentId === null) {
|
|
473
|
-
roots.push(span);
|
|
474
|
-
}
|
|
475
|
-
else {
|
|
476
|
-
if (!childrenMap[parentId]) {
|
|
477
|
-
childrenMap[parentId] = [];
|
|
478
|
-
}
|
|
479
|
-
childrenMap[parentId].push(span);
|
|
480
|
-
}
|
|
481
|
-
}
|
|
482
|
-
// Sort using parsed dates
|
|
483
|
-
roots.sort((a, b) => Date.parse(a.created_at) - Date.parse(b.created_at));
|
|
484
|
-
for (const parentId in childrenMap) {
|
|
485
|
-
// Sort using parsed dates
|
|
486
|
-
childrenMap[parentId].sort((a, b) => Date.parse(a.created_at) - Date.parse(b.created_at));
|
|
487
|
-
}
|
|
488
|
-
function buildFlatListDfs(span) {
|
|
489
|
-
if (visited.has(span.span_id))
|
|
490
|
-
return;
|
|
491
|
-
visited.add(span.span_id);
|
|
492
|
-
sortedCondensedList.push(span);
|
|
493
|
-
const children = childrenMap[span.span_id] || [];
|
|
494
|
-
for (const child of children) {
|
|
495
|
-
buildFlatListDfs(child);
|
|
496
|
-
}
|
|
497
|
-
}
|
|
498
|
-
for (const rootSpan of roots) {
|
|
499
|
-
buildFlatListDfs(rootSpan);
|
|
500
|
-
}
|
|
501
|
-
for (const span of spansList) {
|
|
502
|
-
if (!visited.has(span.span_id)) {
|
|
503
|
-
console.warn(`Orphaned span detected: ${span.span_id}, adding to end of list.`);
|
|
504
|
-
buildFlatListDfs(span);
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
return sortedCondensedList;
|
|
508
|
-
}
|
|
509
|
-
save() {
|
|
510
|
-
return __awaiter(this, arguments, void 0, function* (emptySave = false) {
|
|
511
|
-
var _a;
|
|
512
|
-
// If monitoring is disabled or trace hasn't started, don't save
|
|
513
|
-
if (!this.enableMonitoring || this.startTime === -1 || !this.traceManager) {
|
|
514
|
-
logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Monitoring disabled or trace not started. Not saving.`);
|
|
515
|
-
return null;
|
|
516
|
-
}
|
|
517
|
-
const endTime = Date.now() / 1000; // Current time in seconds
|
|
518
|
-
const duration = endTime - this.startTime;
|
|
519
|
-
const condensedEntries = this.condenseTrace(getTraceClientContext().entries);
|
|
520
|
-
// Calculate token counts and costs
|
|
521
|
-
const tokenCounts = {
|
|
522
|
-
prompt_tokens: 0,
|
|
523
|
-
completion_tokens: 0,
|
|
524
|
-
total_tokens: 0,
|
|
525
|
-
prompt_tokens_cost_usd: 0.0,
|
|
526
|
-
completion_tokens_cost_usd: 0.0,
|
|
527
|
-
total_cost_usd: 0.0
|
|
528
|
-
};
|
|
529
|
-
// Use a Set to avoid double-counting tokens from nested calls if structure is complex
|
|
530
|
-
// Note: Assuming span_ids are unique across the trace
|
|
531
|
-
const processedSpanIds = new Set();
|
|
532
|
-
for (const entry of condensedEntries) {
|
|
533
|
-
if (processedSpanIds.has(entry.span_id))
|
|
534
|
-
continue;
|
|
535
|
-
processedSpanIds.add(entry.span_id);
|
|
536
|
-
if (entry.span_type === 'llm' && ((_a = entry.output) === null || _a === void 0 ? void 0 : _a.usage)) {
|
|
537
|
-
const usage = entry.output.usage;
|
|
538
|
-
// Sum tokens
|
|
539
|
-
tokenCounts.prompt_tokens += usage.prompt_tokens || usage.input_tokens || 0;
|
|
540
|
-
tokenCounts.completion_tokens += usage.completion_tokens || usage.output_tokens || 0;
|
|
541
|
-
tokenCounts.total_tokens += usage.total_tokens ||
|
|
542
|
-
((usage.prompt_tokens || usage.input_tokens || 0) +
|
|
543
|
-
(usage.completion_tokens || usage.output_tokens || 0));
|
|
544
|
-
// Sum costs embedded in the usage object (if they exist)
|
|
545
|
-
if (usage.prompt_tokens_cost_usd !== undefined) {
|
|
546
|
-
tokenCounts.prompt_tokens_cost_usd += usage.prompt_tokens_cost_usd;
|
|
547
|
-
}
|
|
548
|
-
if (usage.completion_tokens_cost_usd !== undefined) {
|
|
549
|
-
tokenCounts.completion_tokens_cost_usd += usage.completion_tokens_cost_usd;
|
|
550
|
-
}
|
|
551
|
-
if (usage.total_cost_usd !== undefined) {
|
|
552
|
-
tokenCounts.total_cost_usd += usage.total_cost_usd;
|
|
553
|
-
}
|
|
554
|
-
}
|
|
555
|
-
}
|
|
556
|
-
// --- Retrieve and include pending evaluation runs ---
|
|
557
|
-
const evaluationRunsToSave = [...this.pendingEvaluationRuns];
|
|
558
|
-
// ----------------------------------------------------
|
|
559
|
-
const traceData = {
|
|
560
|
-
trace_id: this.traceId,
|
|
561
|
-
name: this.name, // Use the sanitized name
|
|
562
|
-
project_name: this.projectName,
|
|
563
|
-
created_at: new Date(this.startTime * 1000).toISOString(), // Convert start time to ISO string
|
|
564
|
-
duration: duration < 0 ? 0 : duration,
|
|
565
|
-
token_counts: tokenCounts,
|
|
566
|
-
entries: condensedEntries, // Send the potentially nested structure from condenseTrace
|
|
567
|
-
evaluation_runs: evaluationRunsToSave, // <-- ADDED: Include collected evaluations
|
|
568
|
-
overwrite: this.overwrite,
|
|
569
|
-
parent_trace_id: this.parentTraceId,
|
|
570
|
-
parent_name: this.parentName,
|
|
571
|
-
};
|
|
572
|
-
// <<< ADD LOGGING HERE >>>
|
|
573
|
-
logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Payload to be saved:`, JSON.stringify(traceData, null, 2));
|
|
574
|
-
// <<< ADD SPECIFIC LOGGING FOR EVALUATION RUNS (using INFO level) >>>
|
|
575
|
-
logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Evaluation runs included in payload:`, JSON.stringify(traceData.evaluation_runs, null, 2));
|
|
576
|
-
// <<< END SPECIFIC LOGGING >>>
|
|
577
|
-
if (emptySave) {
|
|
578
|
-
// Skip actual saving if emptySave is true (used for context management in generators)
|
|
579
|
-
logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] emptySave=true, skipping actual save call.`);
|
|
580
|
-
return { traceId: this.traceId, traceData };
|
|
581
|
-
}
|
|
582
|
-
try {
|
|
583
|
-
logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Calling traceManager.saveTrace...`);
|
|
584
|
-
const response = yield this.traceManager.saveTrace(traceData);
|
|
585
|
-
logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Trace saved successfully. Response:`, response);
|
|
586
|
-
// Reset trace context after successful save
|
|
587
|
-
getTraceClientContext().entries = [];
|
|
588
|
-
getTraceClientContext().entryStack = [];
|
|
589
|
-
this.pendingEvaluationRuns = []; // <-- ADDED: Clear pending evaluations
|
|
590
|
-
this.startTime = -1; // Reset start time
|
|
591
|
-
return { traceId: this.traceId, traceData: traceData }; // Return payload on success
|
|
592
|
-
}
|
|
593
|
-
catch (error) {
|
|
594
|
-
logger_instance_js_1.default.error(`[TraceClient ${this.traceId}] Error saving trace:`, error);
|
|
595
|
-
// Optionally reset context even on error?
|
|
596
|
-
// getTraceClientContext().entries = [];
|
|
597
|
-
// getTraceClientContext().entryStack = [];
|
|
598
|
-
// this.startTime = -1;
|
|
599
|
-
return null; // Indicate save failure
|
|
600
|
-
}
|
|
601
|
-
});
|
|
602
|
-
}
|
|
603
|
-
print() {
|
|
604
|
-
if (!this.enableMonitoring) {
|
|
605
|
-
// Keep console.log for direct user output when print() is called
|
|
606
|
-
console.log("Monitoring was disabled. No trace entries recorded.");
|
|
607
|
-
return;
|
|
608
|
-
}
|
|
609
|
-
const traceClientContext = getTraceClientContext();
|
|
610
|
-
if (traceClientContext.entries.length === 0) {
|
|
611
|
-
// Keep console.log for direct user output when print() is called
|
|
612
|
-
console.log("No trace entries recorded.");
|
|
613
|
-
return;
|
|
614
|
-
}
|
|
615
|
-
// Keep console.log for direct user output when print() is called
|
|
616
|
-
console.log(`\n--- Trace Details: ${this.name} (ID: ${this.traceId}) ---`);
|
|
617
|
-
traceClientContext.entries.forEach(entry => {
|
|
618
|
-
var _a;
|
|
619
|
-
const indent = " ".repeat((_a = entry.depth) !== null && _a !== void 0 ? _a : 0);
|
|
620
|
-
const timeStr = entry.created_at ? `@ ${new Date(entry.created_at * 1000).toISOString()}` : '';
|
|
621
|
-
const shortSpanId = entry.span_id ? `(id: ${entry.span_id.substring(0, 8)}...)` : '';
|
|
622
|
-
const shortParentId = entry.parent_span_id ? `(parent: ${entry.parent_span_id.substring(0, 8)}...)` : '';
|
|
623
|
-
try {
|
|
624
|
-
switch (entry.type) {
|
|
625
|
-
case 'enter':
|
|
626
|
-
console.log(`${indent}→ ${entry.function || 'unknown'} ${shortSpanId} ${shortParentId} [${entry.span_type || 'span'}] ${timeStr}`);
|
|
627
|
-
break;
|
|
628
|
-
case 'exit':
|
|
629
|
-
const durationStr = entry.duration !== undefined ? `(${entry.duration.toFixed(3)}s)` : '';
|
|
630
|
-
// Keep console.log
|
|
631
|
-
console.log(`${indent}← ${entry.function || 'unknown'} ${shortSpanId} ${durationStr} ${timeStr}`);
|
|
632
|
-
break;
|
|
633
|
-
case 'input':
|
|
634
|
-
let inputStr = JSON.stringify(entry.inputs);
|
|
635
|
-
if (inputStr && inputStr.length > 200) {
|
|
636
|
-
inputStr = inputStr.substring(0, 197) + '...';
|
|
637
|
-
}
|
|
638
|
-
// Keep console.log
|
|
639
|
-
console.log(`${indent} Input (for ${shortSpanId}): ${inputStr || '{}'}`);
|
|
640
|
-
break;
|
|
641
|
-
case 'output':
|
|
642
|
-
case 'error':
|
|
643
|
-
let outputStr = JSON.stringify(entry.output);
|
|
644
|
-
if (outputStr && outputStr.length > 200) {
|
|
645
|
-
outputStr = outputStr.substring(0, 197) + '...';
|
|
646
|
-
}
|
|
647
|
-
const prefix = entry.type === 'error' ? 'Error' : 'Output';
|
|
648
|
-
// Keep console.log
|
|
649
|
-
console.log(`${indent} ${prefix} (for ${shortSpanId}): ${outputStr || 'null'}`);
|
|
650
|
-
break;
|
|
651
|
-
}
|
|
652
|
-
}
|
|
653
|
-
catch (stringifyError) {
|
|
654
|
-
const errorMessage = stringifyError instanceof Error ? stringifyError.message : String(stringifyError);
|
|
655
|
-
// Keep console.log
|
|
656
|
-
console.log(`${indent}! Error formatting entry: ${errorMessage}`);
|
|
657
|
-
console.log(`${indent} Raw entry:`, entry);
|
|
658
|
-
}
|
|
659
|
-
});
|
|
660
|
-
// Keep console.log
|
|
661
|
-
console.log(`--- End Trace: ${this.name} ---`);
|
|
662
|
-
}
|
|
663
|
-
delete() {
|
|
664
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
665
|
-
if (!this.enableMonitoring || !this.traceManager) {
|
|
666
|
-
logger_instance_js_1.default.warn(`Cannot delete trace ${this.traceId}, monitoring disabled or manager missing.`);
|
|
667
|
-
return null;
|
|
668
|
-
}
|
|
669
|
-
try {
|
|
670
|
-
const result = yield this.traceManager.deleteTrace(this.traceId);
|
|
671
|
-
logger_instance_js_1.default.info(`Trace ${this.traceId} deleted successfully.`);
|
|
672
|
-
return result;
|
|
673
|
-
}
|
|
674
|
-
catch (error) {
|
|
675
|
-
logger_instance_js_1.default.error(`Failed to delete trace ${this.traceId}.`, { error: error instanceof Error ? error.message : String(error) });
|
|
676
|
-
throw error; // Re-throw after logging
|
|
677
|
-
}
|
|
678
|
-
});
|
|
679
|
-
}
|
|
680
|
-
/**
|
|
681
|
-
* Asynchronously evaluate an example using the provided scorers,
|
|
682
|
-
* embedding the evaluation request into the trace data.
|
|
683
|
-
* Ported from the Python SDK's async_evaluate method.
|
|
684
|
-
*
|
|
685
|
-
* @param scorers Array of scorers to use for evaluation
|
|
686
|
-
* @param options Evaluation options including input, outputs, and metadata
|
|
687
|
-
* @returns Promise that resolves when the evaluation entry has been added to the trace
|
|
688
|
-
*/
|
|
689
|
-
asyncEvaluate(scorers_1) {
|
|
690
|
-
return __awaiter(this, arguments, void 0, function* (scorers, options = {}) {
|
|
691
|
-
if (!this.enableEvaluations) {
|
|
692
|
-
logger_instance_js_1.default.warn("Evaluations are disabled. Skipping async evaluation.");
|
|
693
|
-
return;
|
|
694
|
-
}
|
|
695
|
-
if (!scorers || scorers.length === 0) {
|
|
696
|
-
logger_instance_js_1.default.warn("No scorers provided. Skipping async evaluation.");
|
|
697
|
-
return;
|
|
698
|
-
}
|
|
699
|
-
// Filter for APIJudgmentScorers as the backend evaluation needs them
|
|
700
|
-
const apiScorers = scorers.filter((s) => s instanceof base_scorer_js_1.APIJudgmentScorer);
|
|
701
|
-
if (apiScorers.length === 0) {
|
|
702
|
-
logger_instance_js_1.default.warn("No APIJudgmentScorers found in the provided scorers list. Skipping async evaluation as backend requires API scorers.");
|
|
703
|
-
return;
|
|
704
|
-
}
|
|
705
|
-
// Process rules (currently just using this.rules directly)
|
|
706
|
-
const loadedRules = this.rules; // TODO: Add ScorerWrapper-like processing if needed in TS
|
|
707
|
-
// Prevent using JudgevalScorer with rules - only APIJudgmentScorer allowed with rules
|
|
708
|
-
if (loadedRules && loadedRules.length > 0 && scorers.some(s => !(s instanceof base_scorer_js_1.APIJudgmentScorer))) {
|
|
709
|
-
throw new Error("Cannot use Judgeval scorers, you can only use API scorers when using rules. Please either remove rules or use only APIJudgmentScorer types.");
|
|
710
|
-
}
|
|
711
|
-
// Create example structure matching Python/backend expectations
|
|
712
|
-
const example = {
|
|
713
|
-
input: options.input || "",
|
|
714
|
-
actual_output: options.actualOutput || "",
|
|
715
|
-
expected_output: options.expectedOutput || "",
|
|
716
|
-
context: options.context || [],
|
|
717
|
-
retrieval_context: options.retrievalContext || [],
|
|
718
|
-
tools_called: options.toolsCalled || [],
|
|
719
|
-
expected_tools: options.expectedTools || [],
|
|
720
|
-
additional_metadata: options.additionalMetadata || {},
|
|
721
|
-
trace_id: this.traceId
|
|
722
|
-
};
|
|
723
|
-
try {
|
|
724
|
-
const traceClientContext = getTraceClientContext();
|
|
725
|
-
const currentEntry = traceClientContext.entryStack.at(-1);
|
|
726
|
-
if (!currentEntry) {
|
|
727
|
-
logger_instance_js_1.default.warn(`No current entry to associate evaluation with\\nStack trace: ${new Error().stack}`);
|
|
728
|
-
return;
|
|
729
|
-
}
|
|
730
|
-
const currentSpanId = currentEntry.span_id; // Get the span ID
|
|
731
|
-
// --- Create evaluation run name (similar to Python) ---
|
|
732
|
-
// Capitalize scorer names
|
|
733
|
-
const scorerNames = apiScorers.map(scorer => {
|
|
734
|
-
var _a;
|
|
735
|
-
// Attempt to get score_type, fallback to class name or Unknown
|
|
736
|
-
const name = (scorer === null || scorer === void 0 ? void 0 : scorer.scoreType) || ((_a = scorer === null || scorer === void 0 ? void 0 : scorer.constructor) === null || _a === void 0 ? void 0 : _a.name) || "Unknown";
|
|
737
|
-
return name.charAt(0).toUpperCase() + name.slice(1);
|
|
738
|
-
}).join(',');
|
|
739
|
-
// Use trace name and shortened span ID (or trace ID if no span)
|
|
740
|
-
const idPart = currentSpanId ? currentSpanId.substring(0, 8) : this.traceId.substring(0, 8);
|
|
741
|
-
const evalName = `${this.name.charAt(0).toUpperCase() + this.name.slice(1)}-${idPart}-[${scorerNames}]`;
|
|
742
|
-
// --- End eval name creation ---
|
|
743
|
-
// Construct the evaluation payload
|
|
744
|
-
const evalRunPayload = {
|
|
745
|
-
organization_id: this.organizationId,
|
|
746
|
-
log_results: options.logResults !== false, // Default to true
|
|
747
|
-
project_name: this.projectName,
|
|
748
|
-
eval_name: evalName,
|
|
749
|
-
examples: [example],
|
|
750
|
-
scorers: apiScorers, // Use the filtered list of API scorers
|
|
751
|
-
model: options.model || "",
|
|
752
|
-
metadata: {}, // Matches Python tracer
|
|
753
|
-
judgment_api_key: this.apiKey,
|
|
754
|
-
override: this.overwrite, // Use trace's overwrite setting
|
|
755
|
-
rules: loadedRules, // Pass the processed rules
|
|
756
|
-
trace_span_id: currentSpanId // <<< RENAMED: Assign the current span ID (matching backend)
|
|
757
|
-
};
|
|
758
|
-
// --- Log the payload before storing (using INFO level) ---
|
|
759
|
-
logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Storing EvaluationRunPayload for span ${currentSpanId}:`, JSON.stringify(evalRunPayload, null, 2));
|
|
760
|
-
// --- End log before store ---
|
|
761
|
-
// --- Store the payload instead of sending ---
|
|
762
|
-
this.pendingEvaluationRuns.push(evalRunPayload);
|
|
763
|
-
logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Evaluation payload stored for span ${currentSpanId}`);
|
|
764
|
-
// --- End store payload ---
|
|
765
|
-
}
|
|
766
|
-
catch (error) {
|
|
767
|
-
logger_instance_js_1.default.error(`Failed during asyncEvaluate execution: ${error instanceof Error ? error.message : String(error)}`);
|
|
768
|
-
throw error; // Re-throw after logging
|
|
769
|
-
}
|
|
770
|
-
});
|
|
771
|
-
}
|
|
772
|
-
// OPTIONAL: Add a method to get the original name if needed elsewhere
|
|
773
|
-
getOriginalName() {
|
|
774
|
-
return this.originalName;
|
|
775
|
-
}
|
|
776
|
-
}
|
|
777
|
-
exports.TraceClient = TraceClient;
|
|
778
|
-
/**
|
|
779
|
-
* Singleton Tracer class. Manages overall tracing configuration and trace creation.
|
|
780
|
-
*/
|
|
781
|
-
class Tracer {
|
|
782
|
-
constructor(config) {
|
|
783
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p;
|
|
784
|
-
this.initialized = false;
|
|
785
|
-
const isNode = typeof process !== 'undefined' && process.versions != null && process.versions.node != null;
|
|
786
|
-
const envApiKey = isNode ? (_a = process.env) === null || _a === void 0 ? void 0 : _a.JUDGMENT_API_KEY : undefined;
|
|
787
|
-
const envOrgId = isNode ? (_b = process.env) === null || _b === void 0 ? void 0 : _b.JUDGMENT_ORG_ID : undefined;
|
|
788
|
-
const envProjectName = isNode ? (_c = process.env) === null || _c === void 0 ? void 0 : _c.JUDGMENT_PROJECT_NAME : undefined;
|
|
789
|
-
const envMonitoring = isNode ? (_d = process.env) === null || _d === void 0 ? void 0 : _d.JUDGMENT_MONITORING : 'true';
|
|
790
|
-
const envEvaluations = isNode ? (_e = process.env) === null || _e === void 0 ? void 0 : _e.JUDGMENT_EVALUATIONS : 'true';
|
|
791
|
-
this.apiKey = (_g = (_f = config === null || config === void 0 ? void 0 : config.apiKey) !== null && _f !== void 0 ? _f : envApiKey) !== null && _g !== void 0 ? _g : '';
|
|
792
|
-
this.organizationId = (_j = (_h = config === null || config === void 0 ? void 0 : config.organizationId) !== null && _h !== void 0 ? _h : envOrgId) !== null && _j !== void 0 ? _j : '';
|
|
793
|
-
this.projectName = (_l = (_k = config === null || config === void 0 ? void 0 : config.projectName) !== null && _k !== void 0 ? _k : envProjectName) !== null && _l !== void 0 ? _l : 'default_project';
|
|
794
|
-
this.defaultRules = (_m = config === null || config === void 0 ? void 0 : config.rules) !== null && _m !== void 0 ? _m : [];
|
|
795
|
-
let effectiveMonitoring = (_o = config === null || config === void 0 ? void 0 : config.enableMonitoring) !== null && _o !== void 0 ? _o : ((envMonitoring === null || envMonitoring === void 0 ? void 0 : envMonitoring.toLowerCase()) !== 'false');
|
|
796
|
-
if (effectiveMonitoring && (!this.apiKey || !this.organizationId)) {
|
|
797
|
-
console.warn("JUDGMENT_API_KEY or JUDGMENT_ORG_ID missing. Monitoring disabled.");
|
|
798
|
-
effectiveMonitoring = false;
|
|
799
|
-
}
|
|
800
|
-
this.enableMonitoring = effectiveMonitoring;
|
|
801
|
-
this.enableEvaluations = effectiveMonitoring && ((_p = config === null || config === void 0 ? void 0 : config.enableEvaluations) !== null && _p !== void 0 ? _p : ((envEvaluations === null || envEvaluations === void 0 ? void 0 : envEvaluations.toLowerCase()) !== 'false'));
|
|
802
|
-
this.initialized = true;
|
|
803
|
-
}
|
|
804
|
-
static getInstance(config) {
|
|
805
|
-
if (!Tracer.instance) {
|
|
806
|
-
Tracer.instance = new Tracer(config);
|
|
807
|
-
}
|
|
808
|
-
else if (config && !Tracer.instance.initialized) {
|
|
809
|
-
console.warn("Tracer getInstance called with config after implicit initialization. Re-initializing.");
|
|
810
|
-
Tracer.instance = new Tracer(config);
|
|
811
|
-
}
|
|
812
|
-
else if (config && Tracer.instance.initialized) {
|
|
813
|
-
if (config.projectName && config.projectName !== Tracer.instance.projectName) {
|
|
814
|
-
console.warn(`Attempting to re-initialize Tracer with different project_name. Original will be used: '${Tracer.instance.projectName}'.`);
|
|
815
|
-
}
|
|
816
|
-
if (config.rules && config.rules.length > 0 && Tracer.instance.defaultRules.length === 0) {
|
|
817
|
-
try {
|
|
818
|
-
console.warn("Setting default rules on Tracer instance after initial creation.");
|
|
819
|
-
Tracer.instance.defaultRules = config.rules;
|
|
820
|
-
}
|
|
821
|
-
catch (e) {
|
|
822
|
-
console.error("Failed to set default rules after tracer initialization:", e);
|
|
823
|
-
}
|
|
824
|
-
}
|
|
825
|
-
else if (config.rules && JSON.stringify(config.rules) !== JSON.stringify(Tracer.instance.defaultRules)) {
|
|
826
|
-
console.warn("Attempting to change default rules on Tracer after initialization. Original rules will be used.");
|
|
827
|
-
}
|
|
828
|
-
}
|
|
829
|
-
return Tracer.instance;
|
|
830
|
-
}
|
|
831
|
-
getCurrentTrace() {
|
|
832
|
-
return this.currentTrace;
|
|
833
|
-
}
|
|
834
|
-
startTrace(name, config) {
|
|
835
|
-
var _a, _b;
|
|
836
|
-
const parentTrace = this.getCurrentTrace();
|
|
837
|
-
const projectName = (_a = config.projectName) !== null && _a !== void 0 ? _a : this.projectName;
|
|
838
|
-
const effectiveRules = Object.values(Object.fromEntries([...this.defaultRules, ...((_b = config.rules) !== null && _b !== void 0 ? _b : [])]
|
|
839
|
-
.map(rule => { var _a; return [(_a = rule.rule_id) !== null && _a !== void 0 ? _a : rule.name, rule]; })));
|
|
840
|
-
const traceClient = new TraceClient({
|
|
841
|
-
tracer: this,
|
|
842
|
-
name: name,
|
|
843
|
-
projectName: projectName,
|
|
844
|
-
overwrite: config.overwrite,
|
|
845
|
-
rules: effectiveRules,
|
|
846
|
-
enableMonitoring: this.enableMonitoring,
|
|
847
|
-
enableEvaluations: this.enableEvaluations,
|
|
848
|
-
parentTraceId: parentTrace === null || parentTrace === void 0 ? void 0 : parentTrace.traceId,
|
|
849
|
-
parentName: parentTrace === null || parentTrace === void 0 ? void 0 : parentTrace.name,
|
|
850
|
-
apiKey: this.apiKey,
|
|
851
|
-
organizationId: this.organizationId,
|
|
852
|
-
});
|
|
853
|
-
return traceClient;
|
|
854
|
-
}
|
|
855
|
-
*trace(name, options = {}) {
|
|
856
|
-
var _a;
|
|
857
|
-
const trace = this.startTrace(name, Object.assign({}, options));
|
|
858
|
-
const shouldCreateRootSpan = (_a = options.createRootSpan) !== null && _a !== void 0 ? _a : true;
|
|
859
|
-
const prevTrace = this.currentTrace;
|
|
860
|
-
if (shouldCreateRootSpan) {
|
|
861
|
-
for (const span of trace.span(name, { spanType: 'chain' })) {
|
|
862
|
-
this.currentTrace = trace;
|
|
863
|
-
yield trace;
|
|
864
|
-
this.currentTrace = prevTrace;
|
|
865
|
-
}
|
|
866
|
-
}
|
|
867
|
-
else {
|
|
868
|
-
this.currentTrace = trace;
|
|
869
|
-
yield trace;
|
|
870
|
-
this.currentTrace = prevTrace;
|
|
871
|
-
}
|
|
872
|
-
if (trace.enableMonitoring) {
|
|
873
|
-
trace.save(false).catch(saveErr => {
|
|
874
|
-
console.error(`Failed to save completed trace '${name}' (${trace.traceId}):`, saveErr);
|
|
875
|
-
});
|
|
876
|
-
}
|
|
877
|
-
}
|
|
878
|
-
observe(options) {
|
|
879
|
-
if (!this.enableMonitoring) {
|
|
880
|
-
return (func) => (...args) => Promise.resolve(func(...args));
|
|
881
|
-
}
|
|
882
|
-
return (func) => {
|
|
883
|
-
var _a, _b, _c;
|
|
884
|
-
const spanName = (_b = (_a = options === null || options === void 0 ? void 0 : options.name) !== null && _a !== void 0 ? _a : func.name) !== null && _b !== void 0 ? _b : 'anonymous_function';
|
|
885
|
-
const spanType = (_c = options === null || options === void 0 ? void 0 : options.spanType) !== null && _c !== void 0 ? _c : 'span';
|
|
886
|
-
return asyncFunctionWrapper(((...args) => __awaiter(this, void 0, void 0, function* () {
|
|
887
|
-
const currentTrace = this.getCurrentTrace();
|
|
888
|
-
let output;
|
|
889
|
-
let error;
|
|
890
|
-
if (!currentTrace) {
|
|
891
|
-
for (const trace of this.trace(spanName, { createRootSpan: false })) {
|
|
892
|
-
for (const span of trace.span(spanName, { spanType })) {
|
|
893
|
-
span.recordInput({ args: args });
|
|
894
|
-
try {
|
|
895
|
-
span.recordOutput(output = yield func(...args));
|
|
896
|
-
}
|
|
897
|
-
catch (e) {
|
|
898
|
-
span.recordError(error = e);
|
|
899
|
-
}
|
|
900
|
-
}
|
|
901
|
-
}
|
|
902
|
-
}
|
|
903
|
-
else {
|
|
904
|
-
for (const span of currentTrace.span(spanName, { spanType })) {
|
|
905
|
-
span.recordInput({ args: args });
|
|
906
|
-
try {
|
|
907
|
-
span.recordOutput(output = yield func(...args));
|
|
908
|
-
}
|
|
909
|
-
catch (e) {
|
|
910
|
-
span.recordError(error = e);
|
|
911
|
-
}
|
|
912
|
-
}
|
|
913
|
-
}
|
|
914
|
-
if (error) {
|
|
915
|
-
throw error;
|
|
916
|
-
}
|
|
917
|
-
// @ts-expect-error output is assigned
|
|
918
|
-
return output;
|
|
919
|
-
})).bind(func));
|
|
920
|
-
};
|
|
921
|
-
}
|
|
922
|
-
}
|
|
923
|
-
exports.Tracer = Tracer;
|
|
924
|
-
Tracer.instance = null;
|
|
925
|
-
// --- Helper Functions for Wrapping LLM Clients --- //
|
|
926
|
-
// Return owner and method name for explicit patching
|
|
927
|
-
function _getClientConfig(client) {
|
|
928
|
-
var _a, _b, _c, _d, _e, _f;
|
|
929
|
-
// Check OpenAI structure first
|
|
930
|
-
if (client instanceof openai_1.default && typeof ((_b = (_a = client === null || client === void 0 ? void 0 : client.chat) === null || _a === void 0 ? void 0 : _a.completions) === null || _b === void 0 ? void 0 : _b.create) === 'function') {
|
|
931
|
-
return {
|
|
932
|
-
spanName: "OPENAI_API_CALL",
|
|
933
|
-
originalMethod: client.chat.completions.create,
|
|
934
|
-
methodOwner: client.chat.completions,
|
|
935
|
-
methodName: 'create'
|
|
936
|
-
};
|
|
937
|
-
}
|
|
938
|
-
// Check Anthropic structure next
|
|
939
|
-
else if (client instanceof sdk_1.default && typeof ((_c = client === null || client === void 0 ? void 0 : client.messages) === null || _c === void 0 ? void 0 : _c.create) === 'function') {
|
|
940
|
-
return {
|
|
941
|
-
spanName: "ANTHROPIC_API_CALL",
|
|
942
|
-
originalMethod: client.messages.create,
|
|
943
|
-
methodOwner: client.messages,
|
|
944
|
-
methodName: 'create'
|
|
945
|
-
};
|
|
946
|
-
}
|
|
947
|
-
// Check for Together structure (duck typing - looking for .chat.completions.create for v0.7.0)
|
|
948
|
-
else if (typeof ((_e = (_d = client === null || client === void 0 ? void 0 : client.chat) === null || _d === void 0 ? void 0 : _d.completions) === null || _e === void 0 ? void 0 : _e.create) === 'function') {
|
|
949
|
-
const chatCompletionsObj = client.chat.completions;
|
|
950
|
-
return {
|
|
951
|
-
spanName: "TOGETHER_API_CALL",
|
|
952
|
-
originalMethod: chatCompletionsObj.create,
|
|
953
|
-
methodOwner: chatCompletionsObj, // Owner is chat.completions
|
|
954
|
-
methodName: 'create'
|
|
955
|
-
};
|
|
956
|
-
}
|
|
957
|
-
// Fallback/Warning if none match
|
|
958
|
-
logger_instance_js_1.default.warn("Cannot wrap client: Unsupported type or incompatible SDK structure.", { clientType: (_f = client === null || client === void 0 ? void 0 : client.constructor) === null || _f === void 0 ? void 0 : _f.name });
|
|
959
|
-
return null;
|
|
960
|
-
}
|
|
961
|
-
function _formatInputData(client, args) {
|
|
962
|
-
var _a, _b;
|
|
963
|
-
const params = args[0] || {};
|
|
964
|
-
try {
|
|
965
|
-
// Check for OpenAI or Together (assuming chat structure)
|
|
966
|
-
if (client instanceof openai_1.default || (((_a = client === null || client === void 0 ? void 0 : client.constructor) === null || _a === void 0 ? void 0 : _a.name) === 'Together' && ((_b = client === null || client === void 0 ? void 0 : client.chat) === null || _b === void 0 ? void 0 : _b.completions))) {
|
|
967
|
-
return { model: params.model, messages: params.messages, /* other potential params */ };
|
|
968
|
-
}
|
|
969
|
-
else if (client instanceof sdk_1.default) {
|
|
970
|
-
return { model: params.model, messages: params.messages, max_tokens: params.max_tokens, };
|
|
971
|
-
}
|
|
972
|
-
}
|
|
973
|
-
catch (e) {
|
|
974
|
-
logger_instance_js_1.default.error("Error formatting LLM input:", { error: e instanceof Error ? e.message : String(e), params });
|
|
975
|
-
return { raw_params: params };
|
|
976
|
-
}
|
|
977
|
-
return { raw_params: params };
|
|
978
|
-
}
|
|
979
|
-
function _formatOutputData(client, response) {
|
|
980
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
|
|
981
|
-
try {
|
|
982
|
-
// Separate handling for OpenAI and Together, assuming Together might differ
|
|
983
|
-
if (client instanceof openai_1.default && ((_b = (_a = response === null || response === void 0 ? void 0 : response.choices) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.message)) {
|
|
984
|
-
const message = response.choices[0].message;
|
|
985
|
-
return { content: message.content, usage: response.usage, };
|
|
986
|
-
}
|
|
987
|
-
else if ( // Check for Together (assuming chat structure)
|
|
988
|
-
(((_c = client === null || client === void 0 ? void 0 : client.constructor) === null || _c === void 0 ? void 0 : _c.name) === 'Together' && ((_d = client === null || client === void 0 ? void 0 : client.chat) === null || _d === void 0 ? void 0 : _d.completions)) &&
|
|
989
|
-
((_f = (_e = response === null || response === void 0 ? void 0 : response.choices) === null || _e === void 0 ? void 0 : _e[0]) === null || _f === void 0 ? void 0 : _f.message)) {
|
|
990
|
-
const message = response.choices[0].message;
|
|
991
|
-
// Together API might return usage directly or nested, check defensively
|
|
992
|
-
const usage = (response === null || response === void 0 ? void 0 : response.usage) || ((_h = (_g = response === null || response === void 0 ? void 0 : response.choices) === null || _g === void 0 ? void 0 : _g[0]) === null || _h === void 0 ? void 0 : _h.usage) || null;
|
|
993
|
-
return { content: message === null || message === void 0 ? void 0 : message.content, usage: usage, };
|
|
994
|
-
}
|
|
995
|
-
else if (client instanceof sdk_1.default && ((_j = response === null || response === void 0 ? void 0 : response.content) === null || _j === void 0 ? void 0 : _j[0])) {
|
|
996
|
-
const textContent = response.content.filter((block) => block.type === 'text').map((block) => block.text).join('');
|
|
997
|
-
// Reconstruct usage for Anthropic if needed
|
|
998
|
-
const usage = {
|
|
999
|
-
input_tokens: (_k = response.usage) === null || _k === void 0 ? void 0 : _k.input_tokens,
|
|
1000
|
-
output_tokens: (_l = response.usage) === null || _l === void 0 ? void 0 : _l.output_tokens,
|
|
1001
|
-
total_tokens: (((_m = response.usage) === null || _m === void 0 ? void 0 : _m.input_tokens) || 0) + (((_o = response.usage) === null || _o === void 0 ? void 0 : _o.output_tokens) || 0)
|
|
1002
|
-
};
|
|
1003
|
-
return { content: textContent, usage: usage, };
|
|
1004
|
-
}
|
|
1005
|
-
}
|
|
1006
|
-
catch (e) {
|
|
1007
|
-
logger_instance_js_1.default.error("Error formatting LLM output:", { error: e instanceof Error ? e.message : String(e) });
|
|
1008
|
-
return { formatting_error: String(e), raw_response: response };
|
|
1009
|
-
}
|
|
1010
|
-
// Return raw if structure doesn't match known patterns
|
|
1011
|
-
return { raw_response: response };
|
|
1012
|
-
}
|
|
1013
|
-
// --- The Wrap Function --- //
|
|
1014
|
-
function wrap(client) {
|
|
1015
|
-
var _a, _b, _c;
|
|
1016
|
-
const tracer = Tracer.getInstance();
|
|
1017
|
-
if (!tracer.enableMonitoring) {
|
|
1018
|
-
logger_instance_js_1.default.info("Global monitoring disabled, client wrapping skipped.");
|
|
1019
|
-
return client;
|
|
1020
|
-
}
|
|
1021
|
-
// Generic logic for all clients (OpenAI, Anthropic, Together via duck-typing)
|
|
1022
|
-
const config = _getClientConfig(client);
|
|
1023
|
-
if (!config) {
|
|
1024
|
-
return client; // Warning already logged
|
|
1025
|
-
}
|
|
1026
|
-
const { spanName, originalMethod, methodOwner, methodName } = config;
|
|
1027
|
-
const boundOriginalMethod = originalMethod.bind(methodOwner);
|
|
1028
|
-
const tracedMethod = (...args) => __awaiter(this, void 0, void 0, function* () {
|
|
1029
|
-
var _a, _b, _c, _d, _e;
|
|
1030
|
-
const currentTrace = tracer.getCurrentTrace();
|
|
1031
|
-
if (!currentTrace || !currentTrace.enableMonitoring) {
|
|
1032
|
-
return boundOriginalMethod(...args);
|
|
1033
|
-
}
|
|
1034
|
-
let response;
|
|
1035
|
-
for (const span of currentTrace.span(spanName, { spanType: 'llm' })) {
|
|
1036
|
-
// 1. Record Input (extract data needed for recording)
|
|
1037
|
-
const inputDataForRecording = _formatInputData(client, args);
|
|
1038
|
-
currentTrace.recordInput(inputDataForRecording);
|
|
1039
|
-
try {
|
|
1040
|
-
// 2. Make the API call
|
|
1041
|
-
response = yield boundOriginalMethod(...args);
|
|
1042
|
-
}
|
|
1043
|
-
catch (error) {
|
|
1044
|
-
currentTrace.recordError(error);
|
|
1045
|
-
throw error;
|
|
1046
|
-
}
|
|
1047
|
-
// 3. Extract output data *after* await
|
|
1048
|
-
const outputData = _formatOutputData(client, response);
|
|
1049
|
-
// --- Calculate and Embed Costs (Reorganized) ---
|
|
1050
|
-
// 4. Extract modelName and usage *immediately* before the check
|
|
1051
|
-
// Get modelName directly from the *original arguments* `args`
|
|
1052
|
-
// Get usage from the `outputData` extracted above
|
|
1053
|
-
const modelName = (_a = args[0]) === null || _a === void 0 ? void 0 : _a.model; // More direct extraction from original args
|
|
1054
|
-
const traceManager = currentTrace.traceManager;
|
|
1055
|
-
// 5. Perform the check and cost calculation
|
|
1056
|
-
if (modelName && (response === null || response === void 0 ? void 0 : response.usage) && traceManager &&
|
|
1057
|
-
(response.usage.prompt_tokens !== undefined || response.usage.input_tokens !== undefined) &&
|
|
1058
|
-
(response.usage.completion_tokens !== undefined || response.usage.output_tokens !== undefined)) {
|
|
1059
|
-
// Use response.usage directly for calculations too
|
|
1060
|
-
const currentUsage = response.usage;
|
|
1061
|
-
const promptTokens = (_c = (_b = currentUsage.prompt_tokens) !== null && _b !== void 0 ? _b : currentUsage.input_tokens) !== null && _c !== void 0 ? _c : 0;
|
|
1062
|
-
const completionTokens = (_e = (_d = currentUsage.completion_tokens) !== null && _d !== void 0 ? _d : currentUsage.output_tokens) !== null && _e !== void 0 ? _e : 0;
|
|
1063
|
-
try {
|
|
1064
|
-
const costResponse = yield traceManager.calculateTokenCosts(modelName, promptTokens, completionTokens);
|
|
1065
|
-
if (costResponse) {
|
|
1066
|
-
// Merge costs into the usage object *within outputData*
|
|
1067
|
-
// Need to ensure outputData exists and has usage if we modify it
|
|
1068
|
-
if (outputData) { // Check if outputData was created
|
|
1069
|
-
outputData.usage = Object.assign(Object.assign({}, currentUsage), { prompt_tokens_cost_usd: costResponse.prompt_tokens_cost_usd, completion_tokens_cost_usd: costResponse.completion_tokens_cost_usd, total_cost_usd: costResponse.total_cost_usd });
|
|
1070
|
-
}
|
|
1071
|
-
logger_instance_js_1.default.info(`[wrap] Calculated cost for ${modelName}: $${costResponse.total_cost_usd.toFixed(6)}`);
|
|
1072
|
-
}
|
|
1073
|
-
else {
|
|
1074
|
-
logger_instance_js_1.default.warn(`[wrap] Could not fetch token costs for model ${modelName}. Costs will be missing for this span.`);
|
|
1075
|
-
}
|
|
1076
|
-
}
|
|
1077
|
-
catch (costError) {
|
|
1078
|
-
logger_instance_js_1.default.error(`[wrap] Error fetching token costs for ${modelName}:`, costError);
|
|
1079
|
-
}
|
|
1080
|
-
}
|
|
1081
|
-
// --- End Cost Calculation ---
|
|
1082
|
-
// 6. Record the potentially enriched outputData
|
|
1083
|
-
currentTrace.recordOutput(outputData);
|
|
1084
|
-
} // End span loop
|
|
1085
|
-
// The response is returned outside the loop, after the span context has ended
|
|
1086
|
-
return response;
|
|
1087
|
-
});
|
|
1088
|
-
// Generic patching
|
|
1089
|
-
if (methodOwner && methodName && typeof methodOwner[methodName] === 'function') {
|
|
1090
|
-
methodOwner[methodName] = tracedMethod;
|
|
1091
|
-
logger_instance_js_1.default.info(`Successfully wrapped method '${methodName}' on owner '${((_a = methodOwner.constructor) === null || _a === void 0 ? void 0 : _a.name) || '(unknown)'}' for client: ${(_b = client === null || client === void 0 ? void 0 : client.constructor) === null || _b === void 0 ? void 0 : _b.name}`);
|
|
1092
|
-
}
|
|
1093
|
-
else {
|
|
1094
|
-
logger_instance_js_1.default.error("Failed to apply wrapper: Could not find method owner or method name.", { clientType: (_c = client === null || client === void 0 ? void 0 : client.constructor) === null || _c === void 0 ? void 0 : _c.name, methodName });
|
|
1095
|
-
return client; // Return unwrapped if patching fails
|
|
1096
|
-
}
|
|
1097
|
-
return client;
|
|
1098
|
-
}
|
|
1099
|
-
//# sourceMappingURL=tracer.js.map
|