judgeval 0.2.9 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (408) hide show
  1. package/dist/constants.d.ts +2 -0
  2. package/dist/constants.d.ts.map +1 -0
  3. package/dist/constants.js +2 -0
  4. package/dist/constants.js.map +1 -0
  5. package/dist/data/example-params.d.ts +13 -0
  6. package/dist/data/example-params.d.ts.map +1 -0
  7. package/dist/data/example-params.js +12 -0
  8. package/dist/data/example-params.js.map +1 -0
  9. package/dist/data/example.d.ts +4 -0
  10. package/dist/data/example.d.ts.map +1 -0
  11. package/dist/data/example.js +10 -0
  12. package/dist/data/example.js.map +1 -0
  13. package/dist/data/index.d.ts +3 -0
  14. package/dist/data/index.d.ts.map +1 -0
  15. package/dist/data/index.js +3 -0
  16. package/dist/data/index.js.map +1 -0
  17. package/dist/env.d.ts +6 -0
  18. package/dist/env.d.ts.map +1 -0
  19. package/dist/env.js +13 -0
  20. package/dist/env.js.map +1 -0
  21. package/dist/index.d.ts +9 -0
  22. package/dist/index.d.ts.map +1 -0
  23. package/dist/index.js +2 -0
  24. package/dist/index.js.map +1 -0
  25. package/dist/index.mjs +2 -0
  26. package/dist/index.mjs.map +1 -0
  27. package/dist/index.umd.js +2 -0
  28. package/dist/index.umd.js.map +1 -0
  29. package/dist/internal/api/index.d.ts +23 -0
  30. package/dist/internal/api/index.d.ts.map +1 -0
  31. package/dist/internal/api/index.js +132 -0
  32. package/dist/internal/api/index.js.map +1 -0
  33. package/dist/internal/api/models/BaseScorer.d.ts +23 -0
  34. package/dist/internal/api/models/BaseScorer.d.ts.map +1 -0
  35. package/dist/internal/api/models/BaseScorer.js +6 -0
  36. package/dist/internal/api/models/BaseScorer.js.map +1 -0
  37. package/dist/internal/api/models/EvalResults.d.ts +12 -0
  38. package/dist/internal/api/models/EvalResults.d.ts.map +1 -0
  39. package/dist/internal/api/models/EvalResults.js +6 -0
  40. package/dist/internal/api/models/EvalResults.js.map +1 -0
  41. package/dist/internal/api/models/EvalResultsFetch.d.ts +9 -0
  42. package/dist/internal/api/models/EvalResultsFetch.d.ts.map +1 -0
  43. package/dist/internal/api/models/EvalResultsFetch.js +6 -0
  44. package/dist/internal/api/models/EvalResultsFetch.js.map +1 -0
  45. package/dist/internal/api/models/Example.d.ts +10 -0
  46. package/dist/internal/api/models/Example.d.ts.map +1 -0
  47. package/dist/internal/api/models/Example.js +6 -0
  48. package/dist/internal/api/models/Example.js.map +1 -0
  49. package/dist/internal/api/models/ExampleEvaluationRun.d.ts +20 -0
  50. package/dist/internal/api/models/ExampleEvaluationRun.d.ts.map +1 -0
  51. package/dist/internal/api/models/ExampleEvaluationRun.js +6 -0
  52. package/dist/internal/api/models/ExampleEvaluationRun.js.map +1 -0
  53. package/dist/internal/api/models/FetchPromptScorersRequest.d.ts +8 -0
  54. package/dist/internal/api/models/FetchPromptScorersRequest.d.ts.map +1 -0
  55. package/dist/internal/api/models/FetchPromptScorersRequest.js +6 -0
  56. package/dist/internal/api/models/FetchPromptScorersRequest.js.map +1 -0
  57. package/dist/internal/api/models/FetchPromptScorersResponse.d.ts +9 -0
  58. package/dist/internal/api/models/FetchPromptScorersResponse.d.ts.map +1 -0
  59. package/dist/internal/api/models/FetchPromptScorersResponse.js +6 -0
  60. package/dist/internal/api/models/FetchPromptScorersResponse.js.map +1 -0
  61. package/dist/internal/api/models/OtelTraceSpan.d.ts +33 -0
  62. package/dist/internal/api/models/OtelTraceSpan.d.ts.map +1 -0
  63. package/dist/internal/api/models/OtelTraceSpan.js +6 -0
  64. package/dist/internal/api/models/OtelTraceSpan.js.map +1 -0
  65. package/dist/internal/api/models/PromptScorer.d.ts +14 -0
  66. package/dist/internal/api/models/PromptScorer.d.ts.map +1 -0
  67. package/dist/internal/api/models/PromptScorer.js +6 -0
  68. package/dist/internal/api/models/PromptScorer.js.map +1 -0
  69. package/dist/internal/api/models/ResolveProjectNameRequest.d.ts +8 -0
  70. package/dist/internal/api/models/ResolveProjectNameRequest.d.ts.map +1 -0
  71. package/dist/internal/api/models/ResolveProjectNameRequest.js +6 -0
  72. package/dist/internal/api/models/ResolveProjectNameRequest.js.map +1 -0
  73. package/dist/internal/api/models/ResolveProjectNameResponse.d.ts +8 -0
  74. package/dist/internal/api/models/ResolveProjectNameResponse.d.ts.map +1 -0
  75. package/dist/internal/api/models/ResolveProjectNameResponse.js +6 -0
  76. package/dist/internal/api/models/ResolveProjectNameResponse.js.map +1 -0
  77. package/dist/internal/api/models/SavePromptScorerRequest.d.ts +12 -0
  78. package/dist/internal/api/models/SavePromptScorerRequest.d.ts.map +1 -0
  79. package/dist/internal/api/models/SavePromptScorerRequest.js +6 -0
  80. package/dist/internal/api/models/SavePromptScorerRequest.js.map +1 -0
  81. package/dist/internal/api/models/SavePromptScorerResponse.d.ts +9 -0
  82. package/dist/internal/api/models/SavePromptScorerResponse.d.ts.map +1 -0
  83. package/dist/internal/api/models/SavePromptScorerResponse.js +6 -0
  84. package/dist/internal/api/models/SavePromptScorerResponse.js.map +1 -0
  85. package/dist/internal/api/models/ScorerConfig.d.ts +13 -0
  86. package/dist/internal/api/models/ScorerConfig.d.ts.map +1 -0
  87. package/dist/internal/api/models/ScorerConfig.js +6 -0
  88. package/dist/internal/api/models/ScorerConfig.js.map +1 -0
  89. package/dist/internal/api/models/ScorerData.d.ts +17 -0
  90. package/dist/internal/api/models/ScorerData.d.ts.map +1 -0
  91. package/dist/internal/api/models/ScorerData.js +6 -0
  92. package/dist/internal/api/models/ScorerData.js.map +1 -0
  93. package/dist/internal/api/models/ScorerExistsRequest.d.ts +8 -0
  94. package/dist/internal/api/models/ScorerExistsRequest.d.ts.map +1 -0
  95. package/dist/internal/api/models/ScorerExistsRequest.js +6 -0
  96. package/dist/internal/api/models/ScorerExistsRequest.js.map +1 -0
  97. package/dist/internal/api/models/ScorerExistsResponse.d.ts +8 -0
  98. package/dist/internal/api/models/ScorerExistsResponse.d.ts.map +1 -0
  99. package/dist/internal/api/models/ScorerExistsResponse.js +6 -0
  100. package/dist/internal/api/models/ScorerExistsResponse.js.map +1 -0
  101. package/dist/internal/api/models/ScoringResult.d.ts +17 -0
  102. package/dist/internal/api/models/ScoringResult.d.ts.map +1 -0
  103. package/dist/internal/api/models/ScoringResult.js +6 -0
  104. package/dist/internal/api/models/ScoringResult.js.map +1 -0
  105. package/dist/internal/api/models/TraceEvaluationRun.d.ts +18 -0
  106. package/dist/internal/api/models/TraceEvaluationRun.d.ts.map +1 -0
  107. package/dist/internal/api/models/TraceEvaluationRun.js +6 -0
  108. package/dist/internal/api/models/TraceEvaluationRun.js.map +1 -0
  109. package/dist/internal/api/models.d.ts +24 -0
  110. package/dist/internal/api/models.d.ts.map +1 -0
  111. package/dist/internal/api/models.js +24 -0
  112. package/dist/internal/api/models.js.map +1 -0
  113. package/dist/scorers/api-scorer.d.ts +23 -0
  114. package/dist/scorers/api-scorer.d.ts.map +1 -0
  115. package/dist/scorers/api-scorer.js +36 -0
  116. package/dist/scorers/api-scorer.js.map +1 -0
  117. package/dist/scorers/api_scorers/answer-correctness-scorer.d.ts +11 -0
  118. package/dist/scorers/api_scorers/answer-correctness-scorer.d.ts.map +1 -0
  119. package/dist/scorers/api_scorers/answer-correctness-scorer.js +20 -0
  120. package/dist/scorers/api_scorers/answer-correctness-scorer.js.map +1 -0
  121. package/dist/scorers/api_scorers/answer-relevancy-scorer.d.ts +11 -0
  122. package/dist/scorers/api_scorers/answer-relevancy-scorer.d.ts.map +1 -0
  123. package/dist/scorers/api_scorers/answer-relevancy-scorer.js +19 -0
  124. package/dist/scorers/api_scorers/answer-relevancy-scorer.js.map +1 -0
  125. package/dist/scorers/api_scorers/faithfulness-scorer.d.ts +11 -0
  126. package/dist/scorers/api_scorers/faithfulness-scorer.d.ts.map +1 -0
  127. package/dist/scorers/api_scorers/faithfulness-scorer.js +20 -0
  128. package/dist/scorers/api_scorers/faithfulness-scorer.js.map +1 -0
  129. package/dist/scorers/api_scorers/index.d.ts +2 -0
  130. package/dist/scorers/api_scorers/index.d.ts.map +1 -0
  131. package/dist/scorers/api_scorers/index.js +2 -0
  132. package/dist/scorers/api_scorers/index.js.map +1 -0
  133. package/dist/scorers/api_scorers/instruction-adherence-scorer.d.ts +11 -0
  134. package/dist/scorers/api_scorers/instruction-adherence-scorer.d.ts.map +1 -0
  135. package/dist/scorers/api_scorers/instruction-adherence-scorer.js +20 -0
  136. package/dist/scorers/api_scorers/instruction-adherence-scorer.js.map +1 -0
  137. package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.d.ts +48 -0
  138. package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.d.ts.map +1 -0
  139. package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.js +133 -0
  140. package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.js.map +1 -0
  141. package/dist/scorers/api_scorers/prompt_scorer/index.d.ts +5 -0
  142. package/dist/scorers/api_scorers/prompt_scorer/index.d.ts.map +1 -0
  143. package/dist/scorers/api_scorers/prompt_scorer/index.js +5 -0
  144. package/dist/scorers/api_scorers/prompt_scorer/index.js.map +1 -0
  145. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.d.ts +14 -0
  146. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.d.ts.map +1 -0
  147. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.js +44 -0
  148. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.js.map +1 -0
  149. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.d.ts +7 -0
  150. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.d.ts.map +1 -0
  151. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.js +9 -0
  152. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.js.map +1 -0
  153. package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.d.ts +6 -0
  154. package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.d.ts.map +1 -0
  155. package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.js +8 -0
  156. package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.js.map +1 -0
  157. package/dist/scorers/base-scorer.d.ts +10 -0
  158. package/dist/scorers/base-scorer.d.ts.map +1 -0
  159. package/dist/scorers/base-scorer.js +41 -0
  160. package/dist/scorers/base-scorer.js.map +1 -0
  161. package/dist/scorers/index.d.ts +4 -0
  162. package/dist/scorers/index.d.ts.map +1 -0
  163. package/dist/scorers/index.js +4 -0
  164. package/dist/scorers/index.js.map +1 -0
  165. package/dist/tracer/OpenTelemetryKeys.d.ts +12 -0
  166. package/dist/tracer/OpenTelemetryKeys.d.ts.map +1 -0
  167. package/dist/tracer/OpenTelemetryKeys.js +12 -0
  168. package/dist/tracer/OpenTelemetryKeys.js.map +1 -0
  169. package/dist/tracer/Tracer.d.ts +47 -0
  170. package/dist/tracer/Tracer.d.ts.map +1 -0
  171. package/dist/tracer/Tracer.js +235 -0
  172. package/dist/tracer/Tracer.js.map +1 -0
  173. package/dist/tracer/TracerConfiguration.d.ts +71 -0
  174. package/dist/tracer/TracerConfiguration.d.ts.map +1 -0
  175. package/dist/tracer/TracerConfiguration.js +111 -0
  176. package/dist/tracer/TracerConfiguration.js.map +1 -0
  177. package/dist/tracer/exporters/JudgmentSpanExporter.d.ts +36 -0
  178. package/dist/tracer/exporters/JudgmentSpanExporter.d.ts.map +1 -0
  179. package/dist/tracer/exporters/JudgmentSpanExporter.js +72 -0
  180. package/dist/tracer/exporters/JudgmentSpanExporter.js.map +1 -0
  181. package/dist/tracer/exporters/NoOpSpanExporter.d.ts +12 -0
  182. package/dist/tracer/exporters/NoOpSpanExporter.d.ts.map +1 -0
  183. package/dist/tracer/exporters/NoOpSpanExporter.js +16 -0
  184. package/dist/tracer/exporters/NoOpSpanExporter.js.map +1 -0
  185. package/dist/tracer/exporters/index.d.ts +3 -0
  186. package/dist/tracer/exporters/index.d.ts.map +1 -0
  187. package/dist/tracer/exporters/index.js +3 -0
  188. package/dist/tracer/exporters/index.js.map +1 -0
  189. package/dist/tracer/index.d.ts +5 -0
  190. package/dist/tracer/index.d.ts.map +1 -0
  191. package/dist/tracer/index.js +5 -0
  192. package/dist/tracer/index.js.map +1 -0
  193. package/dist/umd.d.ts +2 -0
  194. package/dist/umd.d.ts.map +1 -0
  195. package/dist/umd.js +2 -0
  196. package/dist/umd.js.map +1 -0
  197. package/dist/utils/annotate.d.ts +2 -0
  198. package/dist/utils/annotate.d.ts.map +1 -0
  199. package/dist/utils/annotate.js +27 -0
  200. package/dist/utils/annotate.js.map +1 -0
  201. package/dist/utils/index.d.ts +5 -0
  202. package/dist/utils/index.d.ts.map +1 -0
  203. package/dist/utils/index.js +5 -0
  204. package/dist/utils/index.js.map +1 -0
  205. package/dist/utils/logger.d.ts +27 -0
  206. package/dist/utils/logger.d.ts.map +1 -0
  207. package/dist/utils/logger.js +70 -0
  208. package/dist/utils/logger.js.map +1 -0
  209. package/dist/utils/types.d.ts +4 -0
  210. package/dist/utils/types.d.ts.map +1 -0
  211. package/dist/utils/types.js +2 -0
  212. package/dist/utils/types.js.map +1 -0
  213. package/dist/utils/validation.d.ts +11 -0
  214. package/dist/utils/validation.d.ts.map +1 -0
  215. package/dist/utils/validation.js +16 -0
  216. package/dist/utils/validation.js.map +1 -0
  217. package/dist/version.d.ts +2 -0
  218. package/dist/version.d.ts.map +1 -0
  219. package/dist/version.js +2 -0
  220. package/dist/version.js.map +1 -0
  221. package/package.json +57 -108
  222. package/LICENSE.md +0 -202
  223. package/README.md +0 -372
  224. package/dist/cjs/clients.js +0 -78
  225. package/dist/cjs/clients.js.map +0 -1
  226. package/dist/cjs/common/integrations/langgraph.js +0 -471
  227. package/dist/cjs/common/integrations/langgraph.js.map +0 -1
  228. package/dist/cjs/common/logger-instance.js +0 -64
  229. package/dist/cjs/common/logger-instance.js.map +0 -1
  230. package/dist/cjs/common/logger.js +0 -225
  231. package/dist/cjs/common/logger.js.map +0 -1
  232. package/dist/cjs/common/token-costs.js +0 -95
  233. package/dist/cjs/common/token-costs.js.map +0 -1
  234. package/dist/cjs/common/tracer.js +0 -1099
  235. package/dist/cjs/common/tracer.js.map +0 -1
  236. package/dist/cjs/constants.js +0 -347
  237. package/dist/cjs/constants.js.map +0 -1
  238. package/dist/cjs/data/datasets/eval-dataset-client.js +0 -394
  239. package/dist/cjs/data/datasets/eval-dataset-client.js.map +0 -1
  240. package/dist/cjs/data/datasets/eval-dataset.js +0 -405
  241. package/dist/cjs/data/datasets/eval-dataset.js.map +0 -1
  242. package/dist/cjs/data/example.js +0 -159
  243. package/dist/cjs/data/example.js.map +0 -1
  244. package/dist/cjs/data/result.js +0 -83
  245. package/dist/cjs/data/result.js.map +0 -1
  246. package/dist/cjs/e2etests/eval-operations.test.js +0 -282
  247. package/dist/cjs/e2etests/eval-operations.test.js.map +0 -1
  248. package/dist/cjs/e2etests/judgee-traces.test.js +0 -278
  249. package/dist/cjs/e2etests/judgee-traces.test.js.map +0 -1
  250. package/dist/cjs/evaluation-run.js +0 -136
  251. package/dist/cjs/evaluation-run.js.map +0 -1
  252. package/dist/cjs/exporters/otel-exporter.js +0 -352
  253. package/dist/cjs/exporters/otel-exporter.js.map +0 -1
  254. package/dist/cjs/index.js +0 -78
  255. package/dist/cjs/index.js.map +0 -1
  256. package/dist/cjs/judges/index.js +0 -237
  257. package/dist/cjs/judges/index.js.map +0 -1
  258. package/dist/cjs/judgment-client.js +0 -798
  259. package/dist/cjs/judgment-client.js.map +0 -1
  260. package/dist/cjs/rules.js +0 -322
  261. package/dist/cjs/rules.js.map +0 -1
  262. package/dist/cjs/run-evaluation.js +0 -700
  263. package/dist/cjs/run-evaluation.js.map +0 -1
  264. package/dist/cjs/scorers/api-scorer.js +0 -304
  265. package/dist/cjs/scorers/api-scorer.js.map +0 -1
  266. package/dist/cjs/scorers/base-scorer.js +0 -293
  267. package/dist/cjs/scorers/base-scorer.js.map +0 -1
  268. package/dist/cjs/scorers/metrics/answer-correctness/answer-correctness.js +0 -562
  269. package/dist/cjs/scorers/metrics/answer-correctness/answer-correctness.js.map +0 -1
  270. package/dist/cjs/scorers/metrics/answer-correctness/index.js +0 -19
  271. package/dist/cjs/scorers/metrics/answer-correctness/index.js.map +0 -1
  272. package/dist/cjs/scorers/metrics/answer-correctness/prompts.js +0 -175
  273. package/dist/cjs/scorers/metrics/answer-correctness/prompts.js.map +0 -1
  274. package/dist/cjs/scorers/metrics/answer-relevancy/answer-relevancy.js +0 -514
  275. package/dist/cjs/scorers/metrics/answer-relevancy/answer-relevancy.js.map +0 -1
  276. package/dist/cjs/scorers/metrics/answer-relevancy/index.js +0 -19
  277. package/dist/cjs/scorers/metrics/answer-relevancy/index.js.map +0 -1
  278. package/dist/cjs/scorers/metrics/answer-relevancy/prompts.js +0 -179
  279. package/dist/cjs/scorers/metrics/answer-relevancy/prompts.js.map +0 -1
  280. package/dist/cjs/scorers/metrics/faithfulness/faithfulness.js +0 -512
  281. package/dist/cjs/scorers/metrics/faithfulness/faithfulness.js.map +0 -1
  282. package/dist/cjs/scorers/metrics/faithfulness/index.js +0 -19
  283. package/dist/cjs/scorers/metrics/faithfulness/index.js.map +0 -1
  284. package/dist/cjs/scorers/metrics/faithfulness/prompts.js +0 -232
  285. package/dist/cjs/scorers/metrics/faithfulness/prompts.js.map +0 -1
  286. package/dist/cjs/scorers/metrics/hallucination/hallucination.js +0 -386
  287. package/dist/cjs/scorers/metrics/hallucination/hallucination.js.map +0 -1
  288. package/dist/cjs/scorers/metrics/hallucination/index.js +0 -11
  289. package/dist/cjs/scorers/metrics/hallucination/index.js.map +0 -1
  290. package/dist/cjs/scorers/metrics/hallucination/prompts.js +0 -106
  291. package/dist/cjs/scorers/metrics/hallucination/prompts.js.map +0 -1
  292. package/dist/cjs/scorers/metrics/instruction-adherence/index.js +0 -19
  293. package/dist/cjs/scorers/metrics/instruction-adherence/index.js.map +0 -1
  294. package/dist/cjs/scorers/metrics/instruction-adherence/instruction-adherence.js +0 -378
  295. package/dist/cjs/scorers/metrics/instruction-adherence/instruction-adherence.js.map +0 -1
  296. package/dist/cjs/scorers/metrics/instruction-adherence/prompts.js +0 -124
  297. package/dist/cjs/scorers/metrics/instruction-adherence/prompts.js.map +0 -1
  298. package/dist/esm/clients.js +0 -47
  299. package/dist/esm/clients.js.map +0 -1
  300. package/dist/esm/common/integrations/langgraph.js +0 -444
  301. package/dist/esm/common/integrations/langgraph.js.map +0 -1
  302. package/dist/esm/common/logger-instance.js +0 -59
  303. package/dist/esm/common/logger-instance.js.map +0 -1
  304. package/dist/esm/common/logger.js +0 -208
  305. package/dist/esm/common/logger.js.map +0 -1
  306. package/dist/esm/common/token-costs.js +0 -91
  307. package/dist/esm/common/token-costs.js.map +0 -1
  308. package/dist/esm/common/tracer.js +0 -1093
  309. package/dist/esm/common/tracer.js.map +0 -1
  310. package/dist/esm/constants.js +0 -341
  311. package/dist/esm/constants.js.map +0 -1
  312. package/dist/esm/data/datasets/eval-dataset-client.js +0 -387
  313. package/dist/esm/data/datasets/eval-dataset-client.js.map +0 -1
  314. package/dist/esm/data/datasets/eval-dataset.js +0 -375
  315. package/dist/esm/data/datasets/eval-dataset.js.map +0 -1
  316. package/dist/esm/data/example.js +0 -154
  317. package/dist/esm/data/example.js.map +0 -1
  318. package/dist/esm/data/result.js +0 -78
  319. package/dist/esm/data/result.js.map +0 -1
  320. package/dist/esm/e2etests/eval-operations.test.js +0 -254
  321. package/dist/esm/e2etests/eval-operations.test.js.map +0 -1
  322. package/dist/esm/e2etests/judgee-traces.test.js +0 -253
  323. package/dist/esm/e2etests/judgee-traces.test.js.map +0 -1
  324. package/dist/esm/evaluation-run.js +0 -132
  325. package/dist/esm/evaluation-run.js.map +0 -1
  326. package/dist/esm/exporters/otel-exporter.js +0 -348
  327. package/dist/esm/exporters/otel-exporter.js.map +0 -1
  328. package/dist/esm/index.js +0 -20
  329. package/dist/esm/index.js.map +0 -1
  330. package/dist/esm/judges/index.js +0 -205
  331. package/dist/esm/judges/index.js.map +0 -1
  332. package/dist/esm/judgment-client.js +0 -768
  333. package/dist/esm/judgment-client.js.map +0 -1
  334. package/dist/esm/rules.js +0 -314
  335. package/dist/esm/rules.js.map +0 -1
  336. package/dist/esm/run-evaluation.js +0 -681
  337. package/dist/esm/run-evaluation.js.map +0 -1
  338. package/dist/esm/scorers/api-scorer.js +0 -286
  339. package/dist/esm/scorers/api-scorer.js.map +0 -1
  340. package/dist/esm/scorers/base-scorer.js +0 -287
  341. package/dist/esm/scorers/base-scorer.js.map +0 -1
  342. package/dist/esm/scorers/metrics/answer-correctness/answer-correctness.js +0 -558
  343. package/dist/esm/scorers/metrics/answer-correctness/answer-correctness.js.map +0 -1
  344. package/dist/esm/scorers/metrics/answer-correctness/index.js +0 -3
  345. package/dist/esm/scorers/metrics/answer-correctness/index.js.map +0 -1
  346. package/dist/esm/scorers/metrics/answer-correctness/prompts.js +0 -171
  347. package/dist/esm/scorers/metrics/answer-correctness/prompts.js.map +0 -1
  348. package/dist/esm/scorers/metrics/answer-relevancy/answer-relevancy.js +0 -510
  349. package/dist/esm/scorers/metrics/answer-relevancy/answer-relevancy.js.map +0 -1
  350. package/dist/esm/scorers/metrics/answer-relevancy/index.js +0 -3
  351. package/dist/esm/scorers/metrics/answer-relevancy/index.js.map +0 -1
  352. package/dist/esm/scorers/metrics/answer-relevancy/prompts.js +0 -175
  353. package/dist/esm/scorers/metrics/answer-relevancy/prompts.js.map +0 -1
  354. package/dist/esm/scorers/metrics/faithfulness/faithfulness.js +0 -508
  355. package/dist/esm/scorers/metrics/faithfulness/faithfulness.js.map +0 -1
  356. package/dist/esm/scorers/metrics/faithfulness/index.js +0 -3
  357. package/dist/esm/scorers/metrics/faithfulness/index.js.map +0 -1
  358. package/dist/esm/scorers/metrics/faithfulness/prompts.js +0 -228
  359. package/dist/esm/scorers/metrics/faithfulness/prompts.js.map +0 -1
  360. package/dist/esm/scorers/metrics/hallucination/hallucination.js +0 -382
  361. package/dist/esm/scorers/metrics/hallucination/hallucination.js.map +0 -1
  362. package/dist/esm/scorers/metrics/hallucination/index.js +0 -3
  363. package/dist/esm/scorers/metrics/hallucination/index.js.map +0 -1
  364. package/dist/esm/scorers/metrics/hallucination/prompts.js +0 -102
  365. package/dist/esm/scorers/metrics/hallucination/prompts.js.map +0 -1
  366. package/dist/esm/scorers/metrics/instruction-adherence/index.js +0 -3
  367. package/dist/esm/scorers/metrics/instruction-adherence/index.js.map +0 -1
  368. package/dist/esm/scorers/metrics/instruction-adherence/instruction-adherence.js +0 -374
  369. package/dist/esm/scorers/metrics/instruction-adherence/instruction-adherence.js.map +0 -1
  370. package/dist/esm/scorers/metrics/instruction-adherence/prompts.js +0 -120
  371. package/dist/esm/scorers/metrics/instruction-adherence/prompts.js.map +0 -1
  372. package/dist/types/clients.d.ts +0 -6
  373. package/dist/types/common/integrations/langgraph.d.ts +0 -39
  374. package/dist/types/common/logger-instance.d.ts +0 -3
  375. package/dist/types/common/logger.d.ts +0 -54
  376. package/dist/types/common/token-costs.d.ts +0 -36
  377. package/dist/types/common/tracer.d.ts +0 -249
  378. package/dist/types/constants.d.ts +0 -51
  379. package/dist/types/data/datasets/eval-dataset-client.d.ts +0 -44
  380. package/dist/types/data/datasets/eval-dataset.d.ts +0 -45
  381. package/dist/types/data/example.d.ts +0 -82
  382. package/dist/types/data/result.d.ts +0 -51
  383. package/dist/types/e2etests/eval-operations.test.d.ts +0 -5
  384. package/dist/types/e2etests/judgee-traces.test.d.ts +0 -5
  385. package/dist/types/evaluation-run.d.ts +0 -44
  386. package/dist/types/exporters/otel-exporter.d.ts +0 -16
  387. package/dist/types/index.d.ts +0 -11
  388. package/dist/types/judges/index.d.ts +0 -50
  389. package/dist/types/judgment-client.d.ts +0 -149
  390. package/dist/types/rules.d.ts +0 -120
  391. package/dist/types/run-evaluation.d.ts +0 -78
  392. package/dist/types/scorers/api-scorer.d.ts +0 -84
  393. package/dist/types/scorers/base-scorer.d.ts +0 -162
  394. package/dist/types/scorers/metrics/answer-correctness/answer-correctness.d.ts +0 -80
  395. package/dist/types/scorers/metrics/answer-correctness/index.d.ts +0 -2
  396. package/dist/types/scorers/metrics/answer-correctness/prompts.d.ts +0 -71
  397. package/dist/types/scorers/metrics/answer-relevancy/answer-relevancy.d.ts +0 -74
  398. package/dist/types/scorers/metrics/answer-relevancy/index.d.ts +0 -2
  399. package/dist/types/scorers/metrics/answer-relevancy/prompts.d.ts +0 -71
  400. package/dist/types/scorers/metrics/faithfulness/faithfulness.d.ts +0 -73
  401. package/dist/types/scorers/metrics/faithfulness/index.d.ts +0 -2
  402. package/dist/types/scorers/metrics/faithfulness/prompts.d.ts +0 -94
  403. package/dist/types/scorers/metrics/hallucination/hallucination.d.ts +0 -63
  404. package/dist/types/scorers/metrics/hallucination/index.d.ts +0 -3
  405. package/dist/types/scorers/metrics/hallucination/prompts.d.ts +0 -63
  406. package/dist/types/scorers/metrics/instruction-adherence/index.d.ts +0 -2
  407. package/dist/types/scorers/metrics/instruction-adherence/instruction-adherence.d.ts +0 -63
  408. package/dist/types/scorers/metrics/instruction-adherence/prompts.d.ts +0 -78
@@ -1,1093 +0,0 @@
1
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
- return new (P || (P = Promise))(function (resolve, reject) {
4
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
- step((generator = generator.apply(thisArg, _arguments || [])).next());
8
- });
9
- };
10
- // Core Node.js imports
11
- import { v4 as uuidv4 } from 'uuid';
12
- import { AsyncLocalStorage } from 'async_hooks';
13
- // Installed SDKs
14
- import OpenAI from 'openai';
15
- import Anthropic from '@anthropic-ai/sdk';
16
- // Local Imports
17
- import { JUDGMENT_TRACES_SAVE_API_URL, JUDGMENT_TRACES_FETCH_API_URL, JUDGMENT_TRACES_DELETE_API_URL, JUDGMENT_CALCULATE_TOKEN_COSTS_API_URL,
18
- // Add other necessary constants if needed
19
- } from '../constants.js';
20
- import { APIJudgmentScorer } from '../scorers/base-scorer.js';
21
- import logger from './logger-instance.js'; // Use the shared winston logger instance
22
- // On their own, span() and trace() are fully synchronous
23
- // The issue is with asynchronous functions
24
- // To address this, I made a wrapper for async functions that'll handle all the context stuff
25
- class TraceClientContext {
26
- constructor() {
27
- this.entries = [];
28
- this.entryStack = [];
29
- }
30
- }
31
- const traceClientContextAsyncLocalStorage = new AsyncLocalStorage();
32
- // We could use .enterWith(), but the documentation advises against it
33
- let rootTraceClientContext = new TraceClientContext();
34
- function getTraceClientContext() {
35
- var _a;
36
- return (_a = traceClientContextAsyncLocalStorage.getStore()) !== null && _a !== void 0 ? _a : rootTraceClientContext;
37
- }
38
- function asyncFunctionWrapper(func) {
39
- return (...args) => __awaiter(this, void 0, void 0, function* () {
40
- const parentTraceClientContext = getTraceClientContext();
41
- const traceClientContext = new TraceClientContext();
42
- const lastEntry = parentTraceClientContext.entryStack.at(-1);
43
- if (lastEntry) {
44
- traceClientContext.entryStack.push(lastEntry);
45
- }
46
- const result = yield traceClientContextAsyncLocalStorage
47
- .run(traceClientContext, () => func(...args));
48
- if (traceClientContext.entries.length > parentTraceClientContext.entries.length) {
49
- [traceClientContext.entries, parentTraceClientContext.entries] =
50
- [parentTraceClientContext.entries, traceClientContext.entries];
51
- }
52
- parentTraceClientContext.entries.push(...traceClientContext.entries);
53
- return result;
54
- });
55
- }
56
- // --- API Interaction Client ---
57
- /**
58
- * Client for interacting with Judgment trace API endpoints.
59
- */
60
- class TraceManagerClient {
61
- constructor(apiKey, organizationId) {
62
- if (!apiKey) {
63
- throw new Error("TraceManagerClient requires a Judgment API key.");
64
- }
65
- if (!organizationId) {
66
- throw new Error("TraceManagerClient requires a Judgment Organization ID.");
67
- }
68
- this.apiKey = apiKey;
69
- this.organizationId = organizationId;
70
- }
71
- _fetch(url_1) {
72
- return __awaiter(this, arguments, void 0, function* (url, options = {}) {
73
- const headers = Object.assign({ 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.apiKey}`, 'X-Organization-Id': this.organizationId }, (options.headers || {}));
74
- try {
75
- // Use isomorphic fetch (available globally in modern Node.js and browsers)
76
- const response = yield fetch(url, Object.assign(Object.assign({}, options), { headers: headers }));
77
- // We will return the response object even if !response.ok
78
- // The caller (e.g., saveTrace) is responsible for checking response.ok or response.status
79
- // Handle cases where the response might be empty (e.g., 204 No Content on DELETE)
80
- if (response.status === 204) {
81
- return null; // Indicate success with no content
82
- }
83
- return response;
84
- }
85
- catch (error) {
86
- console.error(`Network or fetch error during ${options.method || 'GET'} ${url}:`, error);
87
- // Re-throw or handle as appropriate for the application context
88
- throw error;
89
- }
90
- });
91
- }
92
- fetchTrace(traceId) {
93
- return __awaiter(this, void 0, void 0, function* () {
94
- return this._fetch(JUDGMENT_TRACES_FETCH_API_URL, {
95
- method: 'POST',
96
- body: JSON.stringify({ trace_id: traceId }),
97
- });
98
- });
99
- }
100
- saveTrace(traceData) {
101
- return __awaiter(this, void 0, void 0, function* () {
102
- // _fetch now returns the raw response object or throws on network error
103
- const response = yield this._fetch(JUDGMENT_TRACES_SAVE_API_URL, {
104
- method: 'POST',
105
- body: JSON.stringify(traceData), // Stringify directly here again
106
- });
107
- // Check if _fetch threw a network error (caught below) or returned an invalid object
108
- if (!response) {
109
- // This case should ideally be caught by _fetch's catch block, but double-check
110
- throw new Error('Failed to save trace data: No response received from API.');
111
- }
112
- // Now, check the status code on the received response object
113
- if (response.status === 400) {
114
- // Attempt to get error body for more info
115
- const errorBody = yield response.text();
116
- throw new Error(`Failed to save trace data: Check your Trace name for conflicts, set overwrite=True to overwrite existing traces: ${response.status} ${response.statusText || ''} - ${errorBody}`);
117
- }
118
- else if (!response.ok) { // Handles other errors (5xx, 4xx except 400)
119
- const errorBody = yield response.text();
120
- throw new Error(`Failed to save trace data: Status ${response.status} ${response.statusText || '(No status text)'} - ${errorBody}`);
121
- }
122
- // --- Success Path ---
123
- // Optionally log the UI URL (needs JSON parsing)
124
- let responseData = null;
125
- try {
126
- // Handle 204 No Content specifically
127
- if (response.status === 204) {
128
- responseData = null; // Or maybe { success: true }?
129
- }
130
- else {
131
- responseData = yield response.json(); // Parse JSON only on success
132
- }
133
- }
134
- catch (parseError) {
135
- logger.warn("Failed to parse successful API response JSON.", { error: parseError });
136
- // Depending on requirements, maybe throw, maybe return a default success object
137
- throw new Error(`API request succeeded (${response.status}), but failed to parse JSON response.`);
138
- }
139
- if (responseData === null || responseData === void 0 ? void 0 : responseData.ui_results_url) {
140
- console.info(`
141
- 🔍 View trace: ${responseData.ui_results_url}
142
- `);
143
- }
144
- // Return the parsed data (or null for 204)
145
- return responseData;
146
- });
147
- }
148
- deleteTrace(traceId) {
149
- return __awaiter(this, void 0, void 0, function* () {
150
- // Assuming DELETE method is correct based on REST principles for the delete endpoint
151
- return this._fetch(JUDGMENT_TRACES_DELETE_API_URL, {
152
- method: 'DELETE',
153
- body: JSON.stringify({ trace_ids: [traceId] }),
154
- });
155
- });
156
- }
157
- deleteTraces(traceIds) {
158
- return __awaiter(this, void 0, void 0, function* () {
159
- return this._fetch(JUDGMENT_TRACES_DELETE_API_URL, {
160
- method: 'DELETE',
161
- body: JSON.stringify({ trace_ids: traceIds }),
162
- });
163
- });
164
- }
165
- /**
166
- * Calculate token costs directly using the API endpoint.
167
- * This is more accurate than client-side calculation as it uses the most up-to-date pricing.
168
- *
169
- * @param model The model name (e.g. 'gpt-4', 'claude-3-opus-20240229')
170
- * @param promptTokens Number of tokens in the prompt/input
171
- * @param completionTokens Number of tokens in the completion/output
172
- * @returns Object containing token counts and calculated costs in USD
173
- */
174
- calculateTokenCosts(model, promptTokens, completionTokens) {
175
- return __awaiter(this, void 0, void 0, function* () {
176
- try {
177
- // Use the new calculation endpoint
178
- const response = yield this._fetch(JUDGMENT_CALCULATE_TOKEN_COSTS_API_URL, {
179
- method: 'POST',
180
- body: JSON.stringify({
181
- model,
182
- prompt_tokens: promptTokens,
183
- completion_tokens: completionTokens
184
- })
185
- });
186
- // Check if the response is okay and parse JSON
187
- if (response && response.ok) {
188
- const data = yield response.json();
189
- return data;
190
- }
191
- else if (response) {
192
- // Log error if response was not ok
193
- const errorBody = yield response.text();
194
- logger.warn(`API error calculating token costs for model ${model}: ${response.status} ${response.statusText}`, { errorBody });
195
- return null;
196
- }
197
- else {
198
- // Handle cases where _fetch might return null or undefined (though it shouldn't with current implementation)
199
- logger.warn(`No response received when calculating token costs for model ${model}.`);
200
- return null;
201
- }
202
- }
203
- catch (error) {
204
- logger.warn(`Failed to calculate token costs for model ${model}.`, { error: error instanceof Error ? error.message : String(error) });
205
- return null;
206
- }
207
- });
208
- }
209
- }
210
- // --- Helper Functions ---
211
- // Helper function to sanitize names (e.g., replace spaces with underscores)
212
- function sanitizeName(name) {
213
- // Replace spaces with underscores and remove potentially problematic characters
214
- // You can adjust the regex further if other characters cause issues.
215
- return name.replace(/\s+/g, '_').replace(/[^a-zA-Z0-9_.-]/g, '');
216
- }
217
- // --- Core Trace Classes ---
218
- /**
219
- * Represents an ongoing trace context.
220
- */
221
- class TraceClient {
222
- constructor(config) {
223
- var _a, _b, _c, _d, _e;
224
- this.traceManager = null; // Made public for wrap access
225
- this._spanDepths = {}; // Track depth of active spans
226
- this.pendingEvaluationRuns = []; // <-- ADDED: Store pending evaluations
227
- this.traceId = config.traceId || uuidv4();
228
- this.originalName = config.name || 'default_trace'; // Store original
229
- this.name = sanitizeName(this.originalName); // Use sanitized name internally
230
- // If the sanitized name is empty, fallback to a default
231
- if (!this.name) {
232
- console.warn(`Original trace name "${this.originalName}" sanitized to empty string. Using default_trace_${this.traceId.substring(0, 8)}.`);
233
- this.name = `default_trace_${this.traceId.substring(0, 8)}`;
234
- }
235
- this.projectName = (_a = config.projectName) !== null && _a !== void 0 ? _a : config.tracer.projectName;
236
- this.overwrite = (_b = config.overwrite) !== null && _b !== void 0 ? _b : false;
237
- this.rules = (_c = config.rules) !== null && _c !== void 0 ? _c : [];
238
- // Determine effective monitoring status based on tracer and API keys
239
- let effectiveMonitoring = (_d = config.enableMonitoring) !== null && _d !== void 0 ? _d : config.tracer.enableMonitoring;
240
- if (effectiveMonitoring && (!config.apiKey || !config.organizationId)) {
241
- console.warn(`TraceClient ${this.traceId}: Monitoring requires JUDGMENT_API_KEY and JUDGMENT_ORG_ID. Disabling monitoring for this trace.`);
242
- effectiveMonitoring = false;
243
- }
244
- this.enableMonitoring = effectiveMonitoring;
245
- // Evaluations depend on monitoring
246
- this.enableEvaluations = effectiveMonitoring && ((_e = config.enableEvaluations) !== null && _e !== void 0 ? _e : config.tracer.enableEvaluations);
247
- this.parentTraceId = config.parentTraceId;
248
- this.parentName = config.parentName;
249
- this.apiKey = config.apiKey;
250
- this.organizationId = config.organizationId;
251
- this.startTime = Date.now() / 1000;
252
- if (this.enableMonitoring) {
253
- this.traceManager = new TraceManagerClient(this.apiKey, this.organizationId);
254
- }
255
- }
256
- addEntry(entry) {
257
- const traceClientContext = getTraceClientContext();
258
- if (this.enableMonitoring) {
259
- traceClientContext.entries.push(entry);
260
- }
261
- }
262
- recordInput(inputs) {
263
- const traceClientContext = getTraceClientContext();
264
- const currentEntry = traceClientContext.entryStack.at(-1);
265
- if (!currentEntry || !currentEntry.span_id) {
266
- console.warn(`No current entry to record input to\nStack trace: ${new Error().stack}`);
267
- return;
268
- }
269
- this.addEntry({
270
- type: 'input',
271
- span_id: currentEntry.span_id,
272
- inputs,
273
- function: currentEntry.function,
274
- depth: this._spanDepths[currentEntry.span_id],
275
- created_at: Date.now() / 1000,
276
- span_type: currentEntry.span_type,
277
- message: `Inputs to ${currentEntry.function}`
278
- });
279
- }
280
- recordOutput(output) {
281
- const traceClientContext = getTraceClientContext();
282
- const currentEntry = traceClientContext.entryStack.at(-1);
283
- if (!currentEntry || !currentEntry.span_id) {
284
- console.warn(`No current entry to record output to\nStack trace: ${new Error().stack}`);
285
- return;
286
- }
287
- this.addEntry({
288
- type: 'output',
289
- span_id: currentEntry.span_id,
290
- output,
291
- function: currentEntry.function,
292
- depth: this._spanDepths[currentEntry.span_id],
293
- created_at: Date.now() / 1000,
294
- span_type: currentEntry.span_type,
295
- message: `Output from ${currentEntry.function}`
296
- });
297
- }
298
- recordError(error) {
299
- const traceClientContext = getTraceClientContext();
300
- const currentEntry = traceClientContext.entryStack.at(-1);
301
- if (!currentEntry || !currentEntry.span_id) {
302
- console.warn(`No current entry to record error to\nStack trace: ${new Error().stack}`);
303
- return;
304
- }
305
- this.addEntry({
306
- type: 'error',
307
- span_id: currentEntry.span_id,
308
- output: error,
309
- function: currentEntry.function,
310
- depth: this._spanDepths[currentEntry.span_id],
311
- created_at: Date.now() / 1000,
312
- span_type: currentEntry.span_type,
313
- message: `Error from ${currentEntry.function}`
314
- });
315
- }
316
- startSpan(name, options = {}) {
317
- var _a;
318
- const traceClientContext = getTraceClientContext();
319
- const parentEntry = traceClientContext.entryStack.at(-1);
320
- const spanId = uuidv4();
321
- const spanType = (_a = options.spanType) !== null && _a !== void 0 ? _a : 'span';
322
- const startTime = Date.now() / 1000;
323
- let depth = 0, parentSpanId = undefined;
324
- if (parentEntry && parentEntry.span_id) {
325
- depth = this._spanDepths[parentEntry.span_id] + 1;
326
- parentSpanId = parentEntry.span_id;
327
- }
328
- this._spanDepths[spanId] = depth;
329
- const entry = {
330
- type: 'enter',
331
- function: name,
332
- span_id: spanId,
333
- depth: depth,
334
- created_at: startTime,
335
- span_type: spanType,
336
- parent_span_id: parentSpanId,
337
- message: name
338
- };
339
- this.addEntry(entry);
340
- traceClientContext.entryStack.push(entry);
341
- }
342
- endSpan() {
343
- const traceClientContext = getTraceClientContext();
344
- const enterEntry = traceClientContext.entryStack.pop();
345
- if (!enterEntry || !enterEntry.span_id) {
346
- console.warn("No enter entry to end");
347
- return;
348
- }
349
- const endTime = Date.now() / 1000;
350
- const duration = endTime - enterEntry.created_at;
351
- this.addEntry({
352
- type: 'exit',
353
- function: enterEntry.function,
354
- span_id: enterEntry.span_id,
355
- depth: this._spanDepths[enterEntry.span_id],
356
- created_at: endTime,
357
- duration: duration,
358
- span_type: enterEntry.span_type,
359
- message: `← ${enterEntry.function}`
360
- });
361
- // Clean up depth tracking
362
- delete this._spanDepths[enterEntry.span_id];
363
- }
364
- *span(name, options = {}) {
365
- if (!this.enableMonitoring) {
366
- yield this;
367
- }
368
- else {
369
- this.startSpan(name, options);
370
- yield this;
371
- this.endSpan();
372
- }
373
- }
374
- /**
375
- * Retrieves the ID of the currently active span in this trace context.
376
- * Relies on AsyncLocalStorage context established by observe/span.
377
- * @returns {string | undefined} The ID of the current span, or undefined if none is active.
378
- */
379
- getCurrentSpanId() {
380
- const traceClientContext = getTraceClientContext(); // Internal function using AsyncLocalStorage
381
- if (!traceClientContext) {
382
- // Should ideally not happen if called within an observe/trace context
383
- console.warn("[Judgeval] getCurrentSpanId called outside of an active trace context.");
384
- return undefined;
385
- }
386
- const currentEntry = traceClientContext.entryStack.at(-1); // Get the latest 'enter' entry
387
- if (!currentEntry) {
388
- // This might happen if called right at the start of a trace before the first span
389
- // console.warn("[Judgeval] getCurrentSpanId called but span stack is empty.");
390
- return undefined;
391
- }
392
- return currentEntry.span_id;
393
- }
394
- getDuration() {
395
- return (Date.now() / 1000) - this.startTime;
396
- }
397
- condenseTrace(rawEntries) {
398
- var _a, _b, _c, _d, _e;
399
- const spansById = {};
400
- for (const entry of rawEntries) {
401
- const spanId = entry.span_id;
402
- if (!spanId)
403
- continue;
404
- if (!spansById[spanId]) {
405
- spansById[spanId] = {
406
- span_id: spanId,
407
- function: entry.function || 'unknown',
408
- depth: (_a = entry.depth) !== null && _a !== void 0 ? _a : 0,
409
- created_at: new Date(((_b = entry.created_at) !== null && _b !== void 0 ? _b : 0) * 1000).toISOString(), // Convert number to ISO string
410
- trace_id: this.traceId, // Add trace_id
411
- parent_span_id: entry.parent_span_id,
412
- span_type: entry.span_type || 'span',
413
- inputs: null,
414
- output: null,
415
- duration: null,
416
- children: []
417
- };
418
- }
419
- const currentSpanData = spansById[spanId];
420
- switch (entry.type) {
421
- case 'enter':
422
- currentSpanData.function = entry.function || currentSpanData.function;
423
- currentSpanData.depth = (_c = entry.depth) !== null && _c !== void 0 ? _c : currentSpanData.depth;
424
- currentSpanData.created_at = new Date(((_d = entry.created_at) !== null && _d !== void 0 ? _d : 0) * 1000).toISOString(); // Ensure created_at is string on update
425
- currentSpanData.parent_span_id = entry.parent_span_id;
426
- currentSpanData.span_type = entry.span_type || currentSpanData.span_type;
427
- currentSpanData.start_time = entry.created_at; // Keep original number for duration calc
428
- break;
429
- case 'exit':
430
- currentSpanData.duration = (_e = entry.duration) !== null && _e !== void 0 ? _e : currentSpanData.duration;
431
- currentSpanData.end_time = entry.created_at; // Keep original number for duration calc
432
- if (currentSpanData.duration === null && currentSpanData.start_time && currentSpanData.end_time) {
433
- currentSpanData.duration = currentSpanData.end_time - currentSpanData.start_time;
434
- }
435
- break;
436
- case 'input':
437
- if (currentSpanData.inputs === null && entry.inputs) {
438
- currentSpanData.inputs = entry.inputs;
439
- }
440
- else if (typeof currentSpanData.inputs === 'object' && typeof entry.inputs === 'object') {
441
- currentSpanData.inputs = Object.assign(Object.assign({}, currentSpanData.inputs), entry.inputs);
442
- }
443
- break;
444
- case 'output':
445
- case 'error':
446
- currentSpanData.output = entry.output;
447
- break;
448
- }
449
- }
450
- const spansList = Object.values(spansById).map(span => {
451
- if (span.duration === null && span.start_time && span.end_time) {
452
- span.duration = span.end_time - span.start_time;
453
- }
454
- delete span.start_time;
455
- delete span.end_time;
456
- return span;
457
- });
458
- const childrenMap = {};
459
- const roots = [];
460
- const spanMap = {};
461
- const sortedCondensedList = [];
462
- const visited = new Set();
463
- for (const span of spansList) {
464
- spanMap[span.span_id] = span;
465
- const parentId = span.parent_span_id;
466
- if (parentId === undefined || parentId === null) {
467
- roots.push(span);
468
- }
469
- else {
470
- if (!childrenMap[parentId]) {
471
- childrenMap[parentId] = [];
472
- }
473
- childrenMap[parentId].push(span);
474
- }
475
- }
476
- // Sort using parsed dates
477
- roots.sort((a, b) => Date.parse(a.created_at) - Date.parse(b.created_at));
478
- for (const parentId in childrenMap) {
479
- // Sort using parsed dates
480
- childrenMap[parentId].sort((a, b) => Date.parse(a.created_at) - Date.parse(b.created_at));
481
- }
482
- function buildFlatListDfs(span) {
483
- if (visited.has(span.span_id))
484
- return;
485
- visited.add(span.span_id);
486
- sortedCondensedList.push(span);
487
- const children = childrenMap[span.span_id] || [];
488
- for (const child of children) {
489
- buildFlatListDfs(child);
490
- }
491
- }
492
- for (const rootSpan of roots) {
493
- buildFlatListDfs(rootSpan);
494
- }
495
- for (const span of spansList) {
496
- if (!visited.has(span.span_id)) {
497
- console.warn(`Orphaned span detected: ${span.span_id}, adding to end of list.`);
498
- buildFlatListDfs(span);
499
- }
500
- }
501
- return sortedCondensedList;
502
- }
503
- save() {
504
- return __awaiter(this, arguments, void 0, function* (emptySave = false) {
505
- var _a;
506
- // If monitoring is disabled or trace hasn't started, don't save
507
- if (!this.enableMonitoring || this.startTime === -1 || !this.traceManager) {
508
- logger.info(`[TraceClient ${this.traceId}] Monitoring disabled or trace not started. Not saving.`);
509
- return null;
510
- }
511
- const endTime = Date.now() / 1000; // Current time in seconds
512
- const duration = endTime - this.startTime;
513
- const condensedEntries = this.condenseTrace(getTraceClientContext().entries);
514
- // Calculate token counts and costs
515
- const tokenCounts = {
516
- prompt_tokens: 0,
517
- completion_tokens: 0,
518
- total_tokens: 0,
519
- prompt_tokens_cost_usd: 0.0,
520
- completion_tokens_cost_usd: 0.0,
521
- total_cost_usd: 0.0
522
- };
523
- // Use a Set to avoid double-counting tokens from nested calls if structure is complex
524
- // Note: Assuming span_ids are unique across the trace
525
- const processedSpanIds = new Set();
526
- for (const entry of condensedEntries) {
527
- if (processedSpanIds.has(entry.span_id))
528
- continue;
529
- processedSpanIds.add(entry.span_id);
530
- if (entry.span_type === 'llm' && ((_a = entry.output) === null || _a === void 0 ? void 0 : _a.usage)) {
531
- const usage = entry.output.usage;
532
- // Sum tokens
533
- tokenCounts.prompt_tokens += usage.prompt_tokens || usage.input_tokens || 0;
534
- tokenCounts.completion_tokens += usage.completion_tokens || usage.output_tokens || 0;
535
- tokenCounts.total_tokens += usage.total_tokens ||
536
- ((usage.prompt_tokens || usage.input_tokens || 0) +
537
- (usage.completion_tokens || usage.output_tokens || 0));
538
- // Sum costs embedded in the usage object (if they exist)
539
- if (usage.prompt_tokens_cost_usd !== undefined) {
540
- tokenCounts.prompt_tokens_cost_usd += usage.prompt_tokens_cost_usd;
541
- }
542
- if (usage.completion_tokens_cost_usd !== undefined) {
543
- tokenCounts.completion_tokens_cost_usd += usage.completion_tokens_cost_usd;
544
- }
545
- if (usage.total_cost_usd !== undefined) {
546
- tokenCounts.total_cost_usd += usage.total_cost_usd;
547
- }
548
- }
549
- }
550
- // --- Retrieve and include pending evaluation runs ---
551
- const evaluationRunsToSave = [...this.pendingEvaluationRuns];
552
- // ----------------------------------------------------
553
- const traceData = {
554
- trace_id: this.traceId,
555
- name: this.name, // Use the sanitized name
556
- project_name: this.projectName,
557
- created_at: new Date(this.startTime * 1000).toISOString(), // Convert start time to ISO string
558
- duration: duration < 0 ? 0 : duration,
559
- token_counts: tokenCounts,
560
- entries: condensedEntries, // Send the potentially nested structure from condenseTrace
561
- evaluation_runs: evaluationRunsToSave, // <-- ADDED: Include collected evaluations
562
- overwrite: this.overwrite,
563
- parent_trace_id: this.parentTraceId,
564
- parent_name: this.parentName,
565
- };
566
- // <<< ADD LOGGING HERE >>>
567
- logger.info(`[TraceClient ${this.traceId}] Payload to be saved:`, JSON.stringify(traceData, null, 2));
568
- // <<< ADD SPECIFIC LOGGING FOR EVALUATION RUNS (using INFO level) >>>
569
- logger.info(`[TraceClient ${this.traceId}] Evaluation runs included in payload:`, JSON.stringify(traceData.evaluation_runs, null, 2));
570
- // <<< END SPECIFIC LOGGING >>>
571
- if (emptySave) {
572
- // Skip actual saving if emptySave is true (used for context management in generators)
573
- logger.info(`[TraceClient ${this.traceId}] emptySave=true, skipping actual save call.`);
574
- return { traceId: this.traceId, traceData };
575
- }
576
- try {
577
- logger.info(`[TraceClient ${this.traceId}] Calling traceManager.saveTrace...`);
578
- const response = yield this.traceManager.saveTrace(traceData);
579
- logger.info(`[TraceClient ${this.traceId}] Trace saved successfully. Response:`, response);
580
- // Reset trace context after successful save
581
- getTraceClientContext().entries = [];
582
- getTraceClientContext().entryStack = [];
583
- this.pendingEvaluationRuns = []; // <-- ADDED: Clear pending evaluations
584
- this.startTime = -1; // Reset start time
585
- return { traceId: this.traceId, traceData: traceData }; // Return payload on success
586
- }
587
- catch (error) {
588
- logger.error(`[TraceClient ${this.traceId}] Error saving trace:`, error);
589
- // Optionally reset context even on error?
590
- // getTraceClientContext().entries = [];
591
- // getTraceClientContext().entryStack = [];
592
- // this.startTime = -1;
593
- return null; // Indicate save failure
594
- }
595
- });
596
- }
597
- print() {
598
- if (!this.enableMonitoring) {
599
- // Keep console.log for direct user output when print() is called
600
- console.log("Monitoring was disabled. No trace entries recorded.");
601
- return;
602
- }
603
- const traceClientContext = getTraceClientContext();
604
- if (traceClientContext.entries.length === 0) {
605
- // Keep console.log for direct user output when print() is called
606
- console.log("No trace entries recorded.");
607
- return;
608
- }
609
- // Keep console.log for direct user output when print() is called
610
- console.log(`\n--- Trace Details: ${this.name} (ID: ${this.traceId}) ---`);
611
- traceClientContext.entries.forEach(entry => {
612
- var _a;
613
- const indent = " ".repeat((_a = entry.depth) !== null && _a !== void 0 ? _a : 0);
614
- const timeStr = entry.created_at ? `@ ${new Date(entry.created_at * 1000).toISOString()}` : '';
615
- const shortSpanId = entry.span_id ? `(id: ${entry.span_id.substring(0, 8)}...)` : '';
616
- const shortParentId = entry.parent_span_id ? `(parent: ${entry.parent_span_id.substring(0, 8)}...)` : '';
617
- try {
618
- switch (entry.type) {
619
- case 'enter':
620
- console.log(`${indent}→ ${entry.function || 'unknown'} ${shortSpanId} ${shortParentId} [${entry.span_type || 'span'}] ${timeStr}`);
621
- break;
622
- case 'exit':
623
- const durationStr = entry.duration !== undefined ? `(${entry.duration.toFixed(3)}s)` : '';
624
- // Keep console.log
625
- console.log(`${indent}← ${entry.function || 'unknown'} ${shortSpanId} ${durationStr} ${timeStr}`);
626
- break;
627
- case 'input':
628
- let inputStr = JSON.stringify(entry.inputs);
629
- if (inputStr && inputStr.length > 200) {
630
- inputStr = inputStr.substring(0, 197) + '...';
631
- }
632
- // Keep console.log
633
- console.log(`${indent} Input (for ${shortSpanId}): ${inputStr || '{}'}`);
634
- break;
635
- case 'output':
636
- case 'error':
637
- let outputStr = JSON.stringify(entry.output);
638
- if (outputStr && outputStr.length > 200) {
639
- outputStr = outputStr.substring(0, 197) + '...';
640
- }
641
- const prefix = entry.type === 'error' ? 'Error' : 'Output';
642
- // Keep console.log
643
- console.log(`${indent} ${prefix} (for ${shortSpanId}): ${outputStr || 'null'}`);
644
- break;
645
- }
646
- }
647
- catch (stringifyError) {
648
- const errorMessage = stringifyError instanceof Error ? stringifyError.message : String(stringifyError);
649
- // Keep console.log
650
- console.log(`${indent}! Error formatting entry: ${errorMessage}`);
651
- console.log(`${indent} Raw entry:`, entry);
652
- }
653
- });
654
- // Keep console.log
655
- console.log(`--- End Trace: ${this.name} ---`);
656
- }
657
- delete() {
658
- return __awaiter(this, void 0, void 0, function* () {
659
- if (!this.enableMonitoring || !this.traceManager) {
660
- logger.warn(`Cannot delete trace ${this.traceId}, monitoring disabled or manager missing.`);
661
- return null;
662
- }
663
- try {
664
- const result = yield this.traceManager.deleteTrace(this.traceId);
665
- logger.info(`Trace ${this.traceId} deleted successfully.`);
666
- return result;
667
- }
668
- catch (error) {
669
- logger.error(`Failed to delete trace ${this.traceId}.`, { error: error instanceof Error ? error.message : String(error) });
670
- throw error; // Re-throw after logging
671
- }
672
- });
673
- }
674
- /**
675
- * Asynchronously evaluate an example using the provided scorers,
676
- * embedding the evaluation request into the trace data.
677
- * Ported from the Python SDK's async_evaluate method.
678
- *
679
- * @param scorers Array of scorers to use for evaluation
680
- * @param options Evaluation options including input, outputs, and metadata
681
- * @returns Promise that resolves when the evaluation entry has been added to the trace
682
- */
683
- asyncEvaluate(scorers_1) {
684
- return __awaiter(this, arguments, void 0, function* (scorers, options = {}) {
685
- if (!this.enableEvaluations) {
686
- logger.warn("Evaluations are disabled. Skipping async evaluation.");
687
- return;
688
- }
689
- if (!scorers || scorers.length === 0) {
690
- logger.warn("No scorers provided. Skipping async evaluation.");
691
- return;
692
- }
693
- // Filter for APIJudgmentScorers as the backend evaluation needs them
694
- const apiScorers = scorers.filter((s) => s instanceof APIJudgmentScorer);
695
- if (apiScorers.length === 0) {
696
- logger.warn("No APIJudgmentScorers found in the provided scorers list. Skipping async evaluation as backend requires API scorers.");
697
- return;
698
- }
699
- // Process rules (currently just using this.rules directly)
700
- const loadedRules = this.rules; // TODO: Add ScorerWrapper-like processing if needed in TS
701
- // Prevent using JudgevalScorer with rules - only APIJudgmentScorer allowed with rules
702
- if (loadedRules && loadedRules.length > 0 && scorers.some(s => !(s instanceof APIJudgmentScorer))) {
703
- throw new Error("Cannot use Judgeval scorers, you can only use API scorers when using rules. Please either remove rules or use only APIJudgmentScorer types.");
704
- }
705
- // Create example structure matching Python/backend expectations
706
- const example = {
707
- input: options.input || "",
708
- actual_output: options.actualOutput || "",
709
- expected_output: options.expectedOutput || "",
710
- context: options.context || [],
711
- retrieval_context: options.retrievalContext || [],
712
- tools_called: options.toolsCalled || [],
713
- expected_tools: options.expectedTools || [],
714
- additional_metadata: options.additionalMetadata || {},
715
- trace_id: this.traceId
716
- };
717
- try {
718
- const traceClientContext = getTraceClientContext();
719
- const currentEntry = traceClientContext.entryStack.at(-1);
720
- if (!currentEntry) {
721
- logger.warn(`No current entry to associate evaluation with\\nStack trace: ${new Error().stack}`);
722
- return;
723
- }
724
- const currentSpanId = currentEntry.span_id; // Get the span ID
725
- // --- Create evaluation run name (similar to Python) ---
726
- // Capitalize scorer names
727
- const scorerNames = apiScorers.map(scorer => {
728
- var _a;
729
- // Attempt to get score_type, fallback to class name or Unknown
730
- const name = (scorer === null || scorer === void 0 ? void 0 : scorer.scoreType) || ((_a = scorer === null || scorer === void 0 ? void 0 : scorer.constructor) === null || _a === void 0 ? void 0 : _a.name) || "Unknown";
731
- return name.charAt(0).toUpperCase() + name.slice(1);
732
- }).join(',');
733
- // Use trace name and shortened span ID (or trace ID if no span)
734
- const idPart = currentSpanId ? currentSpanId.substring(0, 8) : this.traceId.substring(0, 8);
735
- const evalName = `${this.name.charAt(0).toUpperCase() + this.name.slice(1)}-${idPart}-[${scorerNames}]`;
736
- // --- End eval name creation ---
737
- // Construct the evaluation payload
738
- const evalRunPayload = {
739
- organization_id: this.organizationId,
740
- log_results: options.logResults !== false, // Default to true
741
- project_name: this.projectName,
742
- eval_name: evalName,
743
- examples: [example],
744
- scorers: apiScorers, // Use the filtered list of API scorers
745
- model: options.model || "",
746
- metadata: {}, // Matches Python tracer
747
- judgment_api_key: this.apiKey,
748
- override: this.overwrite, // Use trace's overwrite setting
749
- rules: loadedRules, // Pass the processed rules
750
- trace_span_id: currentSpanId // <<< RENAMED: Assign the current span ID (matching backend)
751
- };
752
- // --- Log the payload before storing (using INFO level) ---
753
- logger.info(`[TraceClient ${this.traceId}] Storing EvaluationRunPayload for span ${currentSpanId}:`, JSON.stringify(evalRunPayload, null, 2));
754
- // --- End log before store ---
755
- // --- Store the payload instead of sending ---
756
- this.pendingEvaluationRuns.push(evalRunPayload);
757
- logger.info(`[TraceClient ${this.traceId}] Evaluation payload stored for span ${currentSpanId}`);
758
- // --- End store payload ---
759
- }
760
- catch (error) {
761
- logger.error(`Failed during asyncEvaluate execution: ${error instanceof Error ? error.message : String(error)}`);
762
- throw error; // Re-throw after logging
763
- }
764
- });
765
- }
766
- // OPTIONAL: Add a method to get the original name if needed elsewhere
767
- getOriginalName() {
768
- return this.originalName;
769
- }
770
- }
771
- /**
772
- * Singleton Tracer class. Manages overall tracing configuration and trace creation.
773
- */
774
- class Tracer {
775
- constructor(config) {
776
- var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p;
777
- this.initialized = false;
778
- const isNode = typeof process !== 'undefined' && process.versions != null && process.versions.node != null;
779
- const envApiKey = isNode ? (_a = process.env) === null || _a === void 0 ? void 0 : _a.JUDGMENT_API_KEY : undefined;
780
- const envOrgId = isNode ? (_b = process.env) === null || _b === void 0 ? void 0 : _b.JUDGMENT_ORG_ID : undefined;
781
- const envProjectName = isNode ? (_c = process.env) === null || _c === void 0 ? void 0 : _c.JUDGMENT_PROJECT_NAME : undefined;
782
- const envMonitoring = isNode ? (_d = process.env) === null || _d === void 0 ? void 0 : _d.JUDGMENT_MONITORING : 'true';
783
- const envEvaluations = isNode ? (_e = process.env) === null || _e === void 0 ? void 0 : _e.JUDGMENT_EVALUATIONS : 'true';
784
- this.apiKey = (_g = (_f = config === null || config === void 0 ? void 0 : config.apiKey) !== null && _f !== void 0 ? _f : envApiKey) !== null && _g !== void 0 ? _g : '';
785
- this.organizationId = (_j = (_h = config === null || config === void 0 ? void 0 : config.organizationId) !== null && _h !== void 0 ? _h : envOrgId) !== null && _j !== void 0 ? _j : '';
786
- this.projectName = (_l = (_k = config === null || config === void 0 ? void 0 : config.projectName) !== null && _k !== void 0 ? _k : envProjectName) !== null && _l !== void 0 ? _l : 'default_project';
787
- this.defaultRules = (_m = config === null || config === void 0 ? void 0 : config.rules) !== null && _m !== void 0 ? _m : [];
788
- let effectiveMonitoring = (_o = config === null || config === void 0 ? void 0 : config.enableMonitoring) !== null && _o !== void 0 ? _o : ((envMonitoring === null || envMonitoring === void 0 ? void 0 : envMonitoring.toLowerCase()) !== 'false');
789
- if (effectiveMonitoring && (!this.apiKey || !this.organizationId)) {
790
- console.warn("JUDGMENT_API_KEY or JUDGMENT_ORG_ID missing. Monitoring disabled.");
791
- effectiveMonitoring = false;
792
- }
793
- this.enableMonitoring = effectiveMonitoring;
794
- this.enableEvaluations = effectiveMonitoring && ((_p = config === null || config === void 0 ? void 0 : config.enableEvaluations) !== null && _p !== void 0 ? _p : ((envEvaluations === null || envEvaluations === void 0 ? void 0 : envEvaluations.toLowerCase()) !== 'false'));
795
- this.initialized = true;
796
- }
797
- static getInstance(config) {
798
- if (!Tracer.instance) {
799
- Tracer.instance = new Tracer(config);
800
- }
801
- else if (config && !Tracer.instance.initialized) {
802
- console.warn("Tracer getInstance called with config after implicit initialization. Re-initializing.");
803
- Tracer.instance = new Tracer(config);
804
- }
805
- else if (config && Tracer.instance.initialized) {
806
- if (config.projectName && config.projectName !== Tracer.instance.projectName) {
807
- console.warn(`Attempting to re-initialize Tracer with different project_name. Original will be used: '${Tracer.instance.projectName}'.`);
808
- }
809
- if (config.rules && config.rules.length > 0 && Tracer.instance.defaultRules.length === 0) {
810
- try {
811
- console.warn("Setting default rules on Tracer instance after initial creation.");
812
- Tracer.instance.defaultRules = config.rules;
813
- }
814
- catch (e) {
815
- console.error("Failed to set default rules after tracer initialization:", e);
816
- }
817
- }
818
- else if (config.rules && JSON.stringify(config.rules) !== JSON.stringify(Tracer.instance.defaultRules)) {
819
- console.warn("Attempting to change default rules on Tracer after initialization. Original rules will be used.");
820
- }
821
- }
822
- return Tracer.instance;
823
- }
824
- getCurrentTrace() {
825
- return this.currentTrace;
826
- }
827
- startTrace(name, config) {
828
- var _a, _b;
829
- const parentTrace = this.getCurrentTrace();
830
- const projectName = (_a = config.projectName) !== null && _a !== void 0 ? _a : this.projectName;
831
- const effectiveRules = Object.values(Object.fromEntries([...this.defaultRules, ...((_b = config.rules) !== null && _b !== void 0 ? _b : [])]
832
- .map(rule => { var _a; return [(_a = rule.rule_id) !== null && _a !== void 0 ? _a : rule.name, rule]; })));
833
- const traceClient = new TraceClient({
834
- tracer: this,
835
- name: name,
836
- projectName: projectName,
837
- overwrite: config.overwrite,
838
- rules: effectiveRules,
839
- enableMonitoring: this.enableMonitoring,
840
- enableEvaluations: this.enableEvaluations,
841
- parentTraceId: parentTrace === null || parentTrace === void 0 ? void 0 : parentTrace.traceId,
842
- parentName: parentTrace === null || parentTrace === void 0 ? void 0 : parentTrace.name,
843
- apiKey: this.apiKey,
844
- organizationId: this.organizationId,
845
- });
846
- return traceClient;
847
- }
848
- *trace(name, options = {}) {
849
- var _a;
850
- const trace = this.startTrace(name, Object.assign({}, options));
851
- const shouldCreateRootSpan = (_a = options.createRootSpan) !== null && _a !== void 0 ? _a : true;
852
- const prevTrace = this.currentTrace;
853
- if (shouldCreateRootSpan) {
854
- for (const span of trace.span(name, { spanType: 'chain' })) {
855
- this.currentTrace = trace;
856
- yield trace;
857
- this.currentTrace = prevTrace;
858
- }
859
- }
860
- else {
861
- this.currentTrace = trace;
862
- yield trace;
863
- this.currentTrace = prevTrace;
864
- }
865
- if (trace.enableMonitoring) {
866
- trace.save(false).catch(saveErr => {
867
- console.error(`Failed to save completed trace '${name}' (${trace.traceId}):`, saveErr);
868
- });
869
- }
870
- }
871
- observe(options) {
872
- if (!this.enableMonitoring) {
873
- return (func) => (...args) => Promise.resolve(func(...args));
874
- }
875
- return (func) => {
876
- var _a, _b, _c;
877
- const spanName = (_b = (_a = options === null || options === void 0 ? void 0 : options.name) !== null && _a !== void 0 ? _a : func.name) !== null && _b !== void 0 ? _b : 'anonymous_function';
878
- const spanType = (_c = options === null || options === void 0 ? void 0 : options.spanType) !== null && _c !== void 0 ? _c : 'span';
879
- return asyncFunctionWrapper(((...args) => __awaiter(this, void 0, void 0, function* () {
880
- const currentTrace = this.getCurrentTrace();
881
- let output;
882
- let error;
883
- if (!currentTrace) {
884
- for (const trace of this.trace(spanName, { createRootSpan: false })) {
885
- for (const span of trace.span(spanName, { spanType })) {
886
- span.recordInput({ args: args });
887
- try {
888
- span.recordOutput(output = yield func(...args));
889
- }
890
- catch (e) {
891
- span.recordError(error = e);
892
- }
893
- }
894
- }
895
- }
896
- else {
897
- for (const span of currentTrace.span(spanName, { spanType })) {
898
- span.recordInput({ args: args });
899
- try {
900
- span.recordOutput(output = yield func(...args));
901
- }
902
- catch (e) {
903
- span.recordError(error = e);
904
- }
905
- }
906
- }
907
- if (error) {
908
- throw error;
909
- }
910
- // @ts-expect-error output is assigned
911
- return output;
912
- })).bind(func));
913
- };
914
- }
915
- }
916
- Tracer.instance = null;
917
- // --- Helper Functions for Wrapping LLM Clients --- //
918
- // Return owner and method name for explicit patching
919
- function _getClientConfig(client) {
920
- var _a, _b, _c, _d, _e, _f;
921
- // Check OpenAI structure first
922
- if (client instanceof OpenAI && typeof ((_b = (_a = client === null || client === void 0 ? void 0 : client.chat) === null || _a === void 0 ? void 0 : _a.completions) === null || _b === void 0 ? void 0 : _b.create) === 'function') {
923
- return {
924
- spanName: "OPENAI_API_CALL",
925
- originalMethod: client.chat.completions.create,
926
- methodOwner: client.chat.completions,
927
- methodName: 'create'
928
- };
929
- }
930
- // Check Anthropic structure next
931
- else if (client instanceof Anthropic && typeof ((_c = client === null || client === void 0 ? void 0 : client.messages) === null || _c === void 0 ? void 0 : _c.create) === 'function') {
932
- return {
933
- spanName: "ANTHROPIC_API_CALL",
934
- originalMethod: client.messages.create,
935
- methodOwner: client.messages,
936
- methodName: 'create'
937
- };
938
- }
939
- // Check for Together structure (duck typing - looking for .chat.completions.create for v0.7.0)
940
- else if (typeof ((_e = (_d = client === null || client === void 0 ? void 0 : client.chat) === null || _d === void 0 ? void 0 : _d.completions) === null || _e === void 0 ? void 0 : _e.create) === 'function') {
941
- const chatCompletionsObj = client.chat.completions;
942
- return {
943
- spanName: "TOGETHER_API_CALL",
944
- originalMethod: chatCompletionsObj.create,
945
- methodOwner: chatCompletionsObj, // Owner is chat.completions
946
- methodName: 'create'
947
- };
948
- }
949
- // Fallback/Warning if none match
950
- logger.warn("Cannot wrap client: Unsupported type or incompatible SDK structure.", { clientType: (_f = client === null || client === void 0 ? void 0 : client.constructor) === null || _f === void 0 ? void 0 : _f.name });
951
- return null;
952
- }
953
- function _formatInputData(client, args) {
954
- var _a, _b;
955
- const params = args[0] || {};
956
- try {
957
- // Check for OpenAI or Together (assuming chat structure)
958
- if (client instanceof OpenAI || (((_a = client === null || client === void 0 ? void 0 : client.constructor) === null || _a === void 0 ? void 0 : _a.name) === 'Together' && ((_b = client === null || client === void 0 ? void 0 : client.chat) === null || _b === void 0 ? void 0 : _b.completions))) {
959
- return { model: params.model, messages: params.messages, /* other potential params */ };
960
- }
961
- else if (client instanceof Anthropic) {
962
- return { model: params.model, messages: params.messages, max_tokens: params.max_tokens, };
963
- }
964
- }
965
- catch (e) {
966
- logger.error("Error formatting LLM input:", { error: e instanceof Error ? e.message : String(e), params });
967
- return { raw_params: params };
968
- }
969
- return { raw_params: params };
970
- }
971
- function _formatOutputData(client, response) {
972
- var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
973
- try {
974
- // Separate handling for OpenAI and Together, assuming Together might differ
975
- if (client instanceof OpenAI && ((_b = (_a = response === null || response === void 0 ? void 0 : response.choices) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.message)) {
976
- const message = response.choices[0].message;
977
- return { content: message.content, usage: response.usage, };
978
- }
979
- else if ( // Check for Together (assuming chat structure)
980
- (((_c = client === null || client === void 0 ? void 0 : client.constructor) === null || _c === void 0 ? void 0 : _c.name) === 'Together' && ((_d = client === null || client === void 0 ? void 0 : client.chat) === null || _d === void 0 ? void 0 : _d.completions)) &&
981
- ((_f = (_e = response === null || response === void 0 ? void 0 : response.choices) === null || _e === void 0 ? void 0 : _e[0]) === null || _f === void 0 ? void 0 : _f.message)) {
982
- const message = response.choices[0].message;
983
- // Together API might return usage directly or nested, check defensively
984
- const usage = (response === null || response === void 0 ? void 0 : response.usage) || ((_h = (_g = response === null || response === void 0 ? void 0 : response.choices) === null || _g === void 0 ? void 0 : _g[0]) === null || _h === void 0 ? void 0 : _h.usage) || null;
985
- return { content: message === null || message === void 0 ? void 0 : message.content, usage: usage, };
986
- }
987
- else if (client instanceof Anthropic && ((_j = response === null || response === void 0 ? void 0 : response.content) === null || _j === void 0 ? void 0 : _j[0])) {
988
- const textContent = response.content.filter((block) => block.type === 'text').map((block) => block.text).join('');
989
- // Reconstruct usage for Anthropic if needed
990
- const usage = {
991
- input_tokens: (_k = response.usage) === null || _k === void 0 ? void 0 : _k.input_tokens,
992
- output_tokens: (_l = response.usage) === null || _l === void 0 ? void 0 : _l.output_tokens,
993
- total_tokens: (((_m = response.usage) === null || _m === void 0 ? void 0 : _m.input_tokens) || 0) + (((_o = response.usage) === null || _o === void 0 ? void 0 : _o.output_tokens) || 0)
994
- };
995
- return { content: textContent, usage: usage, };
996
- }
997
- }
998
- catch (e) {
999
- logger.error("Error formatting LLM output:", { error: e instanceof Error ? e.message : String(e) });
1000
- return { formatting_error: String(e), raw_response: response };
1001
- }
1002
- // Return raw if structure doesn't match known patterns
1003
- return { raw_response: response };
1004
- }
1005
- // --- The Wrap Function --- //
1006
- export function wrap(client) {
1007
- var _a, _b, _c;
1008
- const tracer = Tracer.getInstance();
1009
- if (!tracer.enableMonitoring) {
1010
- logger.info("Global monitoring disabled, client wrapping skipped.");
1011
- return client;
1012
- }
1013
- // Generic logic for all clients (OpenAI, Anthropic, Together via duck-typing)
1014
- const config = _getClientConfig(client);
1015
- if (!config) {
1016
- return client; // Warning already logged
1017
- }
1018
- const { spanName, originalMethod, methodOwner, methodName } = config;
1019
- const boundOriginalMethod = originalMethod.bind(methodOwner);
1020
- const tracedMethod = (...args) => __awaiter(this, void 0, void 0, function* () {
1021
- var _a, _b, _c, _d, _e;
1022
- const currentTrace = tracer.getCurrentTrace();
1023
- if (!currentTrace || !currentTrace.enableMonitoring) {
1024
- return boundOriginalMethod(...args);
1025
- }
1026
- let response;
1027
- for (const span of currentTrace.span(spanName, { spanType: 'llm' })) {
1028
- // 1. Record Input (extract data needed for recording)
1029
- const inputDataForRecording = _formatInputData(client, args);
1030
- currentTrace.recordInput(inputDataForRecording);
1031
- try {
1032
- // 2. Make the API call
1033
- response = yield boundOriginalMethod(...args);
1034
- }
1035
- catch (error) {
1036
- currentTrace.recordError(error);
1037
- throw error;
1038
- }
1039
- // 3. Extract output data *after* await
1040
- const outputData = _formatOutputData(client, response);
1041
- // --- Calculate and Embed Costs (Reorganized) ---
1042
- // 4. Extract modelName and usage *immediately* before the check
1043
- // Get modelName directly from the *original arguments* `args`
1044
- // Get usage from the `outputData` extracted above
1045
- const modelName = (_a = args[0]) === null || _a === void 0 ? void 0 : _a.model; // More direct extraction from original args
1046
- const traceManager = currentTrace.traceManager;
1047
- // 5. Perform the check and cost calculation
1048
- if (modelName && (response === null || response === void 0 ? void 0 : response.usage) && traceManager &&
1049
- (response.usage.prompt_tokens !== undefined || response.usage.input_tokens !== undefined) &&
1050
- (response.usage.completion_tokens !== undefined || response.usage.output_tokens !== undefined)) {
1051
- // Use response.usage directly for calculations too
1052
- const currentUsage = response.usage;
1053
- const promptTokens = (_c = (_b = currentUsage.prompt_tokens) !== null && _b !== void 0 ? _b : currentUsage.input_tokens) !== null && _c !== void 0 ? _c : 0;
1054
- const completionTokens = (_e = (_d = currentUsage.completion_tokens) !== null && _d !== void 0 ? _d : currentUsage.output_tokens) !== null && _e !== void 0 ? _e : 0;
1055
- try {
1056
- const costResponse = yield traceManager.calculateTokenCosts(modelName, promptTokens, completionTokens);
1057
- if (costResponse) {
1058
- // Merge costs into the usage object *within outputData*
1059
- // Need to ensure outputData exists and has usage if we modify it
1060
- if (outputData) { // Check if outputData was created
1061
- outputData.usage = Object.assign(Object.assign({}, currentUsage), { prompt_tokens_cost_usd: costResponse.prompt_tokens_cost_usd, completion_tokens_cost_usd: costResponse.completion_tokens_cost_usd, total_cost_usd: costResponse.total_cost_usd });
1062
- }
1063
- logger.info(`[wrap] Calculated cost for ${modelName}: $${costResponse.total_cost_usd.toFixed(6)}`);
1064
- }
1065
- else {
1066
- logger.warn(`[wrap] Could not fetch token costs for model ${modelName}. Costs will be missing for this span.`);
1067
- }
1068
- }
1069
- catch (costError) {
1070
- logger.error(`[wrap] Error fetching token costs for ${modelName}:`, costError);
1071
- }
1072
- }
1073
- // --- End Cost Calculation ---
1074
- // 6. Record the potentially enriched outputData
1075
- currentTrace.recordOutput(outputData);
1076
- } // End span loop
1077
- // The response is returned outside the loop, after the span context has ended
1078
- return response;
1079
- });
1080
- // Generic patching
1081
- if (methodOwner && methodName && typeof methodOwner[methodName] === 'function') {
1082
- methodOwner[methodName] = tracedMethod;
1083
- logger.info(`Successfully wrapped method '${methodName}' on owner '${((_a = methodOwner.constructor) === null || _a === void 0 ? void 0 : _a.name) || '(unknown)'}' for client: ${(_b = client === null || client === void 0 ? void 0 : client.constructor) === null || _b === void 0 ? void 0 : _b.name}`);
1084
- }
1085
- else {
1086
- logger.error("Failed to apply wrapper: Could not find method owner or method name.", { clientType: (_c = client === null || client === void 0 ? void 0 : client.constructor) === null || _c === void 0 ? void 0 : _c.name, methodName });
1087
- return client; // Return unwrapped if patching fails
1088
- }
1089
- return client;
1090
- }
1091
- // --- Exports --- //
1092
- export { Tracer, TraceClient, TraceManagerClient };
1093
- //# sourceMappingURL=tracer.js.map