judgeval 0.2.9 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (408) hide show
  1. package/dist/constants.d.ts +2 -0
  2. package/dist/constants.d.ts.map +1 -0
  3. package/dist/constants.js +2 -0
  4. package/dist/constants.js.map +1 -0
  5. package/dist/data/example-params.d.ts +13 -0
  6. package/dist/data/example-params.d.ts.map +1 -0
  7. package/dist/data/example-params.js +12 -0
  8. package/dist/data/example-params.js.map +1 -0
  9. package/dist/data/example.d.ts +4 -0
  10. package/dist/data/example.d.ts.map +1 -0
  11. package/dist/data/example.js +10 -0
  12. package/dist/data/example.js.map +1 -0
  13. package/dist/data/index.d.ts +3 -0
  14. package/dist/data/index.d.ts.map +1 -0
  15. package/dist/data/index.js +3 -0
  16. package/dist/data/index.js.map +1 -0
  17. package/dist/env.d.ts +6 -0
  18. package/dist/env.d.ts.map +1 -0
  19. package/dist/env.js +13 -0
  20. package/dist/env.js.map +1 -0
  21. package/dist/index.d.ts +9 -0
  22. package/dist/index.d.ts.map +1 -0
  23. package/dist/index.js +2 -0
  24. package/dist/index.js.map +1 -0
  25. package/dist/index.mjs +2 -0
  26. package/dist/index.mjs.map +1 -0
  27. package/dist/index.umd.js +2 -0
  28. package/dist/index.umd.js.map +1 -0
  29. package/dist/internal/api/index.d.ts +23 -0
  30. package/dist/internal/api/index.d.ts.map +1 -0
  31. package/dist/internal/api/index.js +132 -0
  32. package/dist/internal/api/index.js.map +1 -0
  33. package/dist/internal/api/models/BaseScorer.d.ts +23 -0
  34. package/dist/internal/api/models/BaseScorer.d.ts.map +1 -0
  35. package/dist/internal/api/models/BaseScorer.js +6 -0
  36. package/dist/internal/api/models/BaseScorer.js.map +1 -0
  37. package/dist/internal/api/models/EvalResults.d.ts +12 -0
  38. package/dist/internal/api/models/EvalResults.d.ts.map +1 -0
  39. package/dist/internal/api/models/EvalResults.js +6 -0
  40. package/dist/internal/api/models/EvalResults.js.map +1 -0
  41. package/dist/internal/api/models/EvalResultsFetch.d.ts +9 -0
  42. package/dist/internal/api/models/EvalResultsFetch.d.ts.map +1 -0
  43. package/dist/internal/api/models/EvalResultsFetch.js +6 -0
  44. package/dist/internal/api/models/EvalResultsFetch.js.map +1 -0
  45. package/dist/internal/api/models/Example.d.ts +10 -0
  46. package/dist/internal/api/models/Example.d.ts.map +1 -0
  47. package/dist/internal/api/models/Example.js +6 -0
  48. package/dist/internal/api/models/Example.js.map +1 -0
  49. package/dist/internal/api/models/ExampleEvaluationRun.d.ts +20 -0
  50. package/dist/internal/api/models/ExampleEvaluationRun.d.ts.map +1 -0
  51. package/dist/internal/api/models/ExampleEvaluationRun.js +6 -0
  52. package/dist/internal/api/models/ExampleEvaluationRun.js.map +1 -0
  53. package/dist/internal/api/models/FetchPromptScorersRequest.d.ts +8 -0
  54. package/dist/internal/api/models/FetchPromptScorersRequest.d.ts.map +1 -0
  55. package/dist/internal/api/models/FetchPromptScorersRequest.js +6 -0
  56. package/dist/internal/api/models/FetchPromptScorersRequest.js.map +1 -0
  57. package/dist/internal/api/models/FetchPromptScorersResponse.d.ts +9 -0
  58. package/dist/internal/api/models/FetchPromptScorersResponse.d.ts.map +1 -0
  59. package/dist/internal/api/models/FetchPromptScorersResponse.js +6 -0
  60. package/dist/internal/api/models/FetchPromptScorersResponse.js.map +1 -0
  61. package/dist/internal/api/models/OtelTraceSpan.d.ts +33 -0
  62. package/dist/internal/api/models/OtelTraceSpan.d.ts.map +1 -0
  63. package/dist/internal/api/models/OtelTraceSpan.js +6 -0
  64. package/dist/internal/api/models/OtelTraceSpan.js.map +1 -0
  65. package/dist/internal/api/models/PromptScorer.d.ts +14 -0
  66. package/dist/internal/api/models/PromptScorer.d.ts.map +1 -0
  67. package/dist/internal/api/models/PromptScorer.js +6 -0
  68. package/dist/internal/api/models/PromptScorer.js.map +1 -0
  69. package/dist/internal/api/models/ResolveProjectNameRequest.d.ts +8 -0
  70. package/dist/internal/api/models/ResolveProjectNameRequest.d.ts.map +1 -0
  71. package/dist/internal/api/models/ResolveProjectNameRequest.js +6 -0
  72. package/dist/internal/api/models/ResolveProjectNameRequest.js.map +1 -0
  73. package/dist/internal/api/models/ResolveProjectNameResponse.d.ts +8 -0
  74. package/dist/internal/api/models/ResolveProjectNameResponse.d.ts.map +1 -0
  75. package/dist/internal/api/models/ResolveProjectNameResponse.js +6 -0
  76. package/dist/internal/api/models/ResolveProjectNameResponse.js.map +1 -0
  77. package/dist/internal/api/models/SavePromptScorerRequest.d.ts +12 -0
  78. package/dist/internal/api/models/SavePromptScorerRequest.d.ts.map +1 -0
  79. package/dist/internal/api/models/SavePromptScorerRequest.js +6 -0
  80. package/dist/internal/api/models/SavePromptScorerRequest.js.map +1 -0
  81. package/dist/internal/api/models/SavePromptScorerResponse.d.ts +9 -0
  82. package/dist/internal/api/models/SavePromptScorerResponse.d.ts.map +1 -0
  83. package/dist/internal/api/models/SavePromptScorerResponse.js +6 -0
  84. package/dist/internal/api/models/SavePromptScorerResponse.js.map +1 -0
  85. package/dist/internal/api/models/ScorerConfig.d.ts +13 -0
  86. package/dist/internal/api/models/ScorerConfig.d.ts.map +1 -0
  87. package/dist/internal/api/models/ScorerConfig.js +6 -0
  88. package/dist/internal/api/models/ScorerConfig.js.map +1 -0
  89. package/dist/internal/api/models/ScorerData.d.ts +17 -0
  90. package/dist/internal/api/models/ScorerData.d.ts.map +1 -0
  91. package/dist/internal/api/models/ScorerData.js +6 -0
  92. package/dist/internal/api/models/ScorerData.js.map +1 -0
  93. package/dist/internal/api/models/ScorerExistsRequest.d.ts +8 -0
  94. package/dist/internal/api/models/ScorerExistsRequest.d.ts.map +1 -0
  95. package/dist/internal/api/models/ScorerExistsRequest.js +6 -0
  96. package/dist/internal/api/models/ScorerExistsRequest.js.map +1 -0
  97. package/dist/internal/api/models/ScorerExistsResponse.d.ts +8 -0
  98. package/dist/internal/api/models/ScorerExistsResponse.d.ts.map +1 -0
  99. package/dist/internal/api/models/ScorerExistsResponse.js +6 -0
  100. package/dist/internal/api/models/ScorerExistsResponse.js.map +1 -0
  101. package/dist/internal/api/models/ScoringResult.d.ts +17 -0
  102. package/dist/internal/api/models/ScoringResult.d.ts.map +1 -0
  103. package/dist/internal/api/models/ScoringResult.js +6 -0
  104. package/dist/internal/api/models/ScoringResult.js.map +1 -0
  105. package/dist/internal/api/models/TraceEvaluationRun.d.ts +18 -0
  106. package/dist/internal/api/models/TraceEvaluationRun.d.ts.map +1 -0
  107. package/dist/internal/api/models/TraceEvaluationRun.js +6 -0
  108. package/dist/internal/api/models/TraceEvaluationRun.js.map +1 -0
  109. package/dist/internal/api/models.d.ts +24 -0
  110. package/dist/internal/api/models.d.ts.map +1 -0
  111. package/dist/internal/api/models.js +24 -0
  112. package/dist/internal/api/models.js.map +1 -0
  113. package/dist/scorers/api-scorer.d.ts +23 -0
  114. package/dist/scorers/api-scorer.d.ts.map +1 -0
  115. package/dist/scorers/api-scorer.js +36 -0
  116. package/dist/scorers/api-scorer.js.map +1 -0
  117. package/dist/scorers/api_scorers/answer-correctness-scorer.d.ts +11 -0
  118. package/dist/scorers/api_scorers/answer-correctness-scorer.d.ts.map +1 -0
  119. package/dist/scorers/api_scorers/answer-correctness-scorer.js +20 -0
  120. package/dist/scorers/api_scorers/answer-correctness-scorer.js.map +1 -0
  121. package/dist/scorers/api_scorers/answer-relevancy-scorer.d.ts +11 -0
  122. package/dist/scorers/api_scorers/answer-relevancy-scorer.d.ts.map +1 -0
  123. package/dist/scorers/api_scorers/answer-relevancy-scorer.js +19 -0
  124. package/dist/scorers/api_scorers/answer-relevancy-scorer.js.map +1 -0
  125. package/dist/scorers/api_scorers/faithfulness-scorer.d.ts +11 -0
  126. package/dist/scorers/api_scorers/faithfulness-scorer.d.ts.map +1 -0
  127. package/dist/scorers/api_scorers/faithfulness-scorer.js +20 -0
  128. package/dist/scorers/api_scorers/faithfulness-scorer.js.map +1 -0
  129. package/dist/scorers/api_scorers/index.d.ts +2 -0
  130. package/dist/scorers/api_scorers/index.d.ts.map +1 -0
  131. package/dist/scorers/api_scorers/index.js +2 -0
  132. package/dist/scorers/api_scorers/index.js.map +1 -0
  133. package/dist/scorers/api_scorers/instruction-adherence-scorer.d.ts +11 -0
  134. package/dist/scorers/api_scorers/instruction-adherence-scorer.d.ts.map +1 -0
  135. package/dist/scorers/api_scorers/instruction-adherence-scorer.js +20 -0
  136. package/dist/scorers/api_scorers/instruction-adherence-scorer.js.map +1 -0
  137. package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.d.ts +48 -0
  138. package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.d.ts.map +1 -0
  139. package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.js +133 -0
  140. package/dist/scorers/api_scorers/prompt_scorer/base-prompt-scorer.js.map +1 -0
  141. package/dist/scorers/api_scorers/prompt_scorer/index.d.ts +5 -0
  142. package/dist/scorers/api_scorers/prompt_scorer/index.d.ts.map +1 -0
  143. package/dist/scorers/api_scorers/prompt_scorer/index.js +5 -0
  144. package/dist/scorers/api_scorers/prompt_scorer/index.js.map +1 -0
  145. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.d.ts +14 -0
  146. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.d.ts.map +1 -0
  147. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.js +44 -0
  148. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer-utils.js.map +1 -0
  149. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.d.ts +7 -0
  150. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.d.ts.map +1 -0
  151. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.js +9 -0
  152. package/dist/scorers/api_scorers/prompt_scorer/prompt-scorer.js.map +1 -0
  153. package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.d.ts +6 -0
  154. package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.d.ts.map +1 -0
  155. package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.js +8 -0
  156. package/dist/scorers/api_scorers/prompt_scorer/trace-prompt-scorer.js.map +1 -0
  157. package/dist/scorers/base-scorer.d.ts +10 -0
  158. package/dist/scorers/base-scorer.d.ts.map +1 -0
  159. package/dist/scorers/base-scorer.js +41 -0
  160. package/dist/scorers/base-scorer.js.map +1 -0
  161. package/dist/scorers/index.d.ts +4 -0
  162. package/dist/scorers/index.d.ts.map +1 -0
  163. package/dist/scorers/index.js +4 -0
  164. package/dist/scorers/index.js.map +1 -0
  165. package/dist/tracer/OpenTelemetryKeys.d.ts +12 -0
  166. package/dist/tracer/OpenTelemetryKeys.d.ts.map +1 -0
  167. package/dist/tracer/OpenTelemetryKeys.js +12 -0
  168. package/dist/tracer/OpenTelemetryKeys.js.map +1 -0
  169. package/dist/tracer/Tracer.d.ts +47 -0
  170. package/dist/tracer/Tracer.d.ts.map +1 -0
  171. package/dist/tracer/Tracer.js +235 -0
  172. package/dist/tracer/Tracer.js.map +1 -0
  173. package/dist/tracer/TracerConfiguration.d.ts +71 -0
  174. package/dist/tracer/TracerConfiguration.d.ts.map +1 -0
  175. package/dist/tracer/TracerConfiguration.js +111 -0
  176. package/dist/tracer/TracerConfiguration.js.map +1 -0
  177. package/dist/tracer/exporters/JudgmentSpanExporter.d.ts +36 -0
  178. package/dist/tracer/exporters/JudgmentSpanExporter.d.ts.map +1 -0
  179. package/dist/tracer/exporters/JudgmentSpanExporter.js +72 -0
  180. package/dist/tracer/exporters/JudgmentSpanExporter.js.map +1 -0
  181. package/dist/tracer/exporters/NoOpSpanExporter.d.ts +12 -0
  182. package/dist/tracer/exporters/NoOpSpanExporter.d.ts.map +1 -0
  183. package/dist/tracer/exporters/NoOpSpanExporter.js +16 -0
  184. package/dist/tracer/exporters/NoOpSpanExporter.js.map +1 -0
  185. package/dist/tracer/exporters/index.d.ts +3 -0
  186. package/dist/tracer/exporters/index.d.ts.map +1 -0
  187. package/dist/tracer/exporters/index.js +3 -0
  188. package/dist/tracer/exporters/index.js.map +1 -0
  189. package/dist/tracer/index.d.ts +5 -0
  190. package/dist/tracer/index.d.ts.map +1 -0
  191. package/dist/tracer/index.js +5 -0
  192. package/dist/tracer/index.js.map +1 -0
  193. package/dist/umd.d.ts +2 -0
  194. package/dist/umd.d.ts.map +1 -0
  195. package/dist/umd.js +2 -0
  196. package/dist/umd.js.map +1 -0
  197. package/dist/utils/annotate.d.ts +2 -0
  198. package/dist/utils/annotate.d.ts.map +1 -0
  199. package/dist/utils/annotate.js +27 -0
  200. package/dist/utils/annotate.js.map +1 -0
  201. package/dist/utils/index.d.ts +5 -0
  202. package/dist/utils/index.d.ts.map +1 -0
  203. package/dist/utils/index.js +5 -0
  204. package/dist/utils/index.js.map +1 -0
  205. package/dist/utils/logger.d.ts +27 -0
  206. package/dist/utils/logger.d.ts.map +1 -0
  207. package/dist/utils/logger.js +70 -0
  208. package/dist/utils/logger.js.map +1 -0
  209. package/dist/utils/types.d.ts +4 -0
  210. package/dist/utils/types.d.ts.map +1 -0
  211. package/dist/utils/types.js +2 -0
  212. package/dist/utils/types.js.map +1 -0
  213. package/dist/utils/validation.d.ts +11 -0
  214. package/dist/utils/validation.d.ts.map +1 -0
  215. package/dist/utils/validation.js +16 -0
  216. package/dist/utils/validation.js.map +1 -0
  217. package/dist/version.d.ts +2 -0
  218. package/dist/version.d.ts.map +1 -0
  219. package/dist/version.js +2 -0
  220. package/dist/version.js.map +1 -0
  221. package/package.json +57 -108
  222. package/LICENSE.md +0 -202
  223. package/README.md +0 -372
  224. package/dist/cjs/clients.js +0 -78
  225. package/dist/cjs/clients.js.map +0 -1
  226. package/dist/cjs/common/integrations/langgraph.js +0 -471
  227. package/dist/cjs/common/integrations/langgraph.js.map +0 -1
  228. package/dist/cjs/common/logger-instance.js +0 -64
  229. package/dist/cjs/common/logger-instance.js.map +0 -1
  230. package/dist/cjs/common/logger.js +0 -225
  231. package/dist/cjs/common/logger.js.map +0 -1
  232. package/dist/cjs/common/token-costs.js +0 -95
  233. package/dist/cjs/common/token-costs.js.map +0 -1
  234. package/dist/cjs/common/tracer.js +0 -1099
  235. package/dist/cjs/common/tracer.js.map +0 -1
  236. package/dist/cjs/constants.js +0 -347
  237. package/dist/cjs/constants.js.map +0 -1
  238. package/dist/cjs/data/datasets/eval-dataset-client.js +0 -394
  239. package/dist/cjs/data/datasets/eval-dataset-client.js.map +0 -1
  240. package/dist/cjs/data/datasets/eval-dataset.js +0 -405
  241. package/dist/cjs/data/datasets/eval-dataset.js.map +0 -1
  242. package/dist/cjs/data/example.js +0 -159
  243. package/dist/cjs/data/example.js.map +0 -1
  244. package/dist/cjs/data/result.js +0 -83
  245. package/dist/cjs/data/result.js.map +0 -1
  246. package/dist/cjs/e2etests/eval-operations.test.js +0 -282
  247. package/dist/cjs/e2etests/eval-operations.test.js.map +0 -1
  248. package/dist/cjs/e2etests/judgee-traces.test.js +0 -278
  249. package/dist/cjs/e2etests/judgee-traces.test.js.map +0 -1
  250. package/dist/cjs/evaluation-run.js +0 -136
  251. package/dist/cjs/evaluation-run.js.map +0 -1
  252. package/dist/cjs/exporters/otel-exporter.js +0 -352
  253. package/dist/cjs/exporters/otel-exporter.js.map +0 -1
  254. package/dist/cjs/index.js +0 -78
  255. package/dist/cjs/index.js.map +0 -1
  256. package/dist/cjs/judges/index.js +0 -237
  257. package/dist/cjs/judges/index.js.map +0 -1
  258. package/dist/cjs/judgment-client.js +0 -798
  259. package/dist/cjs/judgment-client.js.map +0 -1
  260. package/dist/cjs/rules.js +0 -322
  261. package/dist/cjs/rules.js.map +0 -1
  262. package/dist/cjs/run-evaluation.js +0 -700
  263. package/dist/cjs/run-evaluation.js.map +0 -1
  264. package/dist/cjs/scorers/api-scorer.js +0 -304
  265. package/dist/cjs/scorers/api-scorer.js.map +0 -1
  266. package/dist/cjs/scorers/base-scorer.js +0 -293
  267. package/dist/cjs/scorers/base-scorer.js.map +0 -1
  268. package/dist/cjs/scorers/metrics/answer-correctness/answer-correctness.js +0 -562
  269. package/dist/cjs/scorers/metrics/answer-correctness/answer-correctness.js.map +0 -1
  270. package/dist/cjs/scorers/metrics/answer-correctness/index.js +0 -19
  271. package/dist/cjs/scorers/metrics/answer-correctness/index.js.map +0 -1
  272. package/dist/cjs/scorers/metrics/answer-correctness/prompts.js +0 -175
  273. package/dist/cjs/scorers/metrics/answer-correctness/prompts.js.map +0 -1
  274. package/dist/cjs/scorers/metrics/answer-relevancy/answer-relevancy.js +0 -514
  275. package/dist/cjs/scorers/metrics/answer-relevancy/answer-relevancy.js.map +0 -1
  276. package/dist/cjs/scorers/metrics/answer-relevancy/index.js +0 -19
  277. package/dist/cjs/scorers/metrics/answer-relevancy/index.js.map +0 -1
  278. package/dist/cjs/scorers/metrics/answer-relevancy/prompts.js +0 -179
  279. package/dist/cjs/scorers/metrics/answer-relevancy/prompts.js.map +0 -1
  280. package/dist/cjs/scorers/metrics/faithfulness/faithfulness.js +0 -512
  281. package/dist/cjs/scorers/metrics/faithfulness/faithfulness.js.map +0 -1
  282. package/dist/cjs/scorers/metrics/faithfulness/index.js +0 -19
  283. package/dist/cjs/scorers/metrics/faithfulness/index.js.map +0 -1
  284. package/dist/cjs/scorers/metrics/faithfulness/prompts.js +0 -232
  285. package/dist/cjs/scorers/metrics/faithfulness/prompts.js.map +0 -1
  286. package/dist/cjs/scorers/metrics/hallucination/hallucination.js +0 -386
  287. package/dist/cjs/scorers/metrics/hallucination/hallucination.js.map +0 -1
  288. package/dist/cjs/scorers/metrics/hallucination/index.js +0 -11
  289. package/dist/cjs/scorers/metrics/hallucination/index.js.map +0 -1
  290. package/dist/cjs/scorers/metrics/hallucination/prompts.js +0 -106
  291. package/dist/cjs/scorers/metrics/hallucination/prompts.js.map +0 -1
  292. package/dist/cjs/scorers/metrics/instruction-adherence/index.js +0 -19
  293. package/dist/cjs/scorers/metrics/instruction-adherence/index.js.map +0 -1
  294. package/dist/cjs/scorers/metrics/instruction-adherence/instruction-adherence.js +0 -378
  295. package/dist/cjs/scorers/metrics/instruction-adherence/instruction-adherence.js.map +0 -1
  296. package/dist/cjs/scorers/metrics/instruction-adherence/prompts.js +0 -124
  297. package/dist/cjs/scorers/metrics/instruction-adherence/prompts.js.map +0 -1
  298. package/dist/esm/clients.js +0 -47
  299. package/dist/esm/clients.js.map +0 -1
  300. package/dist/esm/common/integrations/langgraph.js +0 -444
  301. package/dist/esm/common/integrations/langgraph.js.map +0 -1
  302. package/dist/esm/common/logger-instance.js +0 -59
  303. package/dist/esm/common/logger-instance.js.map +0 -1
  304. package/dist/esm/common/logger.js +0 -208
  305. package/dist/esm/common/logger.js.map +0 -1
  306. package/dist/esm/common/token-costs.js +0 -91
  307. package/dist/esm/common/token-costs.js.map +0 -1
  308. package/dist/esm/common/tracer.js +0 -1093
  309. package/dist/esm/common/tracer.js.map +0 -1
  310. package/dist/esm/constants.js +0 -341
  311. package/dist/esm/constants.js.map +0 -1
  312. package/dist/esm/data/datasets/eval-dataset-client.js +0 -387
  313. package/dist/esm/data/datasets/eval-dataset-client.js.map +0 -1
  314. package/dist/esm/data/datasets/eval-dataset.js +0 -375
  315. package/dist/esm/data/datasets/eval-dataset.js.map +0 -1
  316. package/dist/esm/data/example.js +0 -154
  317. package/dist/esm/data/example.js.map +0 -1
  318. package/dist/esm/data/result.js +0 -78
  319. package/dist/esm/data/result.js.map +0 -1
  320. package/dist/esm/e2etests/eval-operations.test.js +0 -254
  321. package/dist/esm/e2etests/eval-operations.test.js.map +0 -1
  322. package/dist/esm/e2etests/judgee-traces.test.js +0 -253
  323. package/dist/esm/e2etests/judgee-traces.test.js.map +0 -1
  324. package/dist/esm/evaluation-run.js +0 -132
  325. package/dist/esm/evaluation-run.js.map +0 -1
  326. package/dist/esm/exporters/otel-exporter.js +0 -348
  327. package/dist/esm/exporters/otel-exporter.js.map +0 -1
  328. package/dist/esm/index.js +0 -20
  329. package/dist/esm/index.js.map +0 -1
  330. package/dist/esm/judges/index.js +0 -205
  331. package/dist/esm/judges/index.js.map +0 -1
  332. package/dist/esm/judgment-client.js +0 -768
  333. package/dist/esm/judgment-client.js.map +0 -1
  334. package/dist/esm/rules.js +0 -314
  335. package/dist/esm/rules.js.map +0 -1
  336. package/dist/esm/run-evaluation.js +0 -681
  337. package/dist/esm/run-evaluation.js.map +0 -1
  338. package/dist/esm/scorers/api-scorer.js +0 -286
  339. package/dist/esm/scorers/api-scorer.js.map +0 -1
  340. package/dist/esm/scorers/base-scorer.js +0 -287
  341. package/dist/esm/scorers/base-scorer.js.map +0 -1
  342. package/dist/esm/scorers/metrics/answer-correctness/answer-correctness.js +0 -558
  343. package/dist/esm/scorers/metrics/answer-correctness/answer-correctness.js.map +0 -1
  344. package/dist/esm/scorers/metrics/answer-correctness/index.js +0 -3
  345. package/dist/esm/scorers/metrics/answer-correctness/index.js.map +0 -1
  346. package/dist/esm/scorers/metrics/answer-correctness/prompts.js +0 -171
  347. package/dist/esm/scorers/metrics/answer-correctness/prompts.js.map +0 -1
  348. package/dist/esm/scorers/metrics/answer-relevancy/answer-relevancy.js +0 -510
  349. package/dist/esm/scorers/metrics/answer-relevancy/answer-relevancy.js.map +0 -1
  350. package/dist/esm/scorers/metrics/answer-relevancy/index.js +0 -3
  351. package/dist/esm/scorers/metrics/answer-relevancy/index.js.map +0 -1
  352. package/dist/esm/scorers/metrics/answer-relevancy/prompts.js +0 -175
  353. package/dist/esm/scorers/metrics/answer-relevancy/prompts.js.map +0 -1
  354. package/dist/esm/scorers/metrics/faithfulness/faithfulness.js +0 -508
  355. package/dist/esm/scorers/metrics/faithfulness/faithfulness.js.map +0 -1
  356. package/dist/esm/scorers/metrics/faithfulness/index.js +0 -3
  357. package/dist/esm/scorers/metrics/faithfulness/index.js.map +0 -1
  358. package/dist/esm/scorers/metrics/faithfulness/prompts.js +0 -228
  359. package/dist/esm/scorers/metrics/faithfulness/prompts.js.map +0 -1
  360. package/dist/esm/scorers/metrics/hallucination/hallucination.js +0 -382
  361. package/dist/esm/scorers/metrics/hallucination/hallucination.js.map +0 -1
  362. package/dist/esm/scorers/metrics/hallucination/index.js +0 -3
  363. package/dist/esm/scorers/metrics/hallucination/index.js.map +0 -1
  364. package/dist/esm/scorers/metrics/hallucination/prompts.js +0 -102
  365. package/dist/esm/scorers/metrics/hallucination/prompts.js.map +0 -1
  366. package/dist/esm/scorers/metrics/instruction-adherence/index.js +0 -3
  367. package/dist/esm/scorers/metrics/instruction-adherence/index.js.map +0 -1
  368. package/dist/esm/scorers/metrics/instruction-adherence/instruction-adherence.js +0 -374
  369. package/dist/esm/scorers/metrics/instruction-adherence/instruction-adherence.js.map +0 -1
  370. package/dist/esm/scorers/metrics/instruction-adherence/prompts.js +0 -120
  371. package/dist/esm/scorers/metrics/instruction-adherence/prompts.js.map +0 -1
  372. package/dist/types/clients.d.ts +0 -6
  373. package/dist/types/common/integrations/langgraph.d.ts +0 -39
  374. package/dist/types/common/logger-instance.d.ts +0 -3
  375. package/dist/types/common/logger.d.ts +0 -54
  376. package/dist/types/common/token-costs.d.ts +0 -36
  377. package/dist/types/common/tracer.d.ts +0 -249
  378. package/dist/types/constants.d.ts +0 -51
  379. package/dist/types/data/datasets/eval-dataset-client.d.ts +0 -44
  380. package/dist/types/data/datasets/eval-dataset.d.ts +0 -45
  381. package/dist/types/data/example.d.ts +0 -82
  382. package/dist/types/data/result.d.ts +0 -51
  383. package/dist/types/e2etests/eval-operations.test.d.ts +0 -5
  384. package/dist/types/e2etests/judgee-traces.test.d.ts +0 -5
  385. package/dist/types/evaluation-run.d.ts +0 -44
  386. package/dist/types/exporters/otel-exporter.d.ts +0 -16
  387. package/dist/types/index.d.ts +0 -11
  388. package/dist/types/judges/index.d.ts +0 -50
  389. package/dist/types/judgment-client.d.ts +0 -149
  390. package/dist/types/rules.d.ts +0 -120
  391. package/dist/types/run-evaluation.d.ts +0 -78
  392. package/dist/types/scorers/api-scorer.d.ts +0 -84
  393. package/dist/types/scorers/base-scorer.d.ts +0 -162
  394. package/dist/types/scorers/metrics/answer-correctness/answer-correctness.d.ts +0 -80
  395. package/dist/types/scorers/metrics/answer-correctness/index.d.ts +0 -2
  396. package/dist/types/scorers/metrics/answer-correctness/prompts.d.ts +0 -71
  397. package/dist/types/scorers/metrics/answer-relevancy/answer-relevancy.d.ts +0 -74
  398. package/dist/types/scorers/metrics/answer-relevancy/index.d.ts +0 -2
  399. package/dist/types/scorers/metrics/answer-relevancy/prompts.d.ts +0 -71
  400. package/dist/types/scorers/metrics/faithfulness/faithfulness.d.ts +0 -73
  401. package/dist/types/scorers/metrics/faithfulness/index.d.ts +0 -2
  402. package/dist/types/scorers/metrics/faithfulness/prompts.d.ts +0 -94
  403. package/dist/types/scorers/metrics/hallucination/hallucination.d.ts +0 -63
  404. package/dist/types/scorers/metrics/hallucination/index.d.ts +0 -3
  405. package/dist/types/scorers/metrics/hallucination/prompts.d.ts +0 -63
  406. package/dist/types/scorers/metrics/instruction-adherence/index.d.ts +0 -2
  407. package/dist/types/scorers/metrics/instruction-adherence/instruction-adherence.d.ts +0 -63
  408. package/dist/types/scorers/metrics/instruction-adherence/prompts.d.ts +0 -78
@@ -1,1099 +0,0 @@
1
- "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __importDefault = (this && this.__importDefault) || function (mod) {
12
- return (mod && mod.__esModule) ? mod : { "default": mod };
13
- };
14
- Object.defineProperty(exports, "__esModule", { value: true });
15
- exports.TraceManagerClient = exports.TraceClient = exports.Tracer = void 0;
16
- exports.wrap = wrap;
17
- // Core Node.js imports
18
- const uuid_1 = require("uuid");
19
- const async_hooks_1 = require("async_hooks");
20
- // Installed SDKs
21
- const openai_1 = __importDefault(require("openai"));
22
- const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
23
- // Local Imports
24
- const constants_js_1 = require("../constants.js");
25
- const base_scorer_js_1 = require("../scorers/base-scorer.js");
26
- const logger_instance_js_1 = __importDefault(require("./logger-instance.js")); // Use the shared winston logger instance
27
- // On their own, span() and trace() are fully synchronous
28
- // The issue is with asynchronous functions
29
- // To address this, I made a wrapper for async functions that'll handle all the context stuff
30
- class TraceClientContext {
31
- constructor() {
32
- this.entries = [];
33
- this.entryStack = [];
34
- }
35
- }
36
- const traceClientContextAsyncLocalStorage = new async_hooks_1.AsyncLocalStorage();
37
- // We could use .enterWith(), but the documentation advises against it
38
- let rootTraceClientContext = new TraceClientContext();
39
- function getTraceClientContext() {
40
- var _a;
41
- return (_a = traceClientContextAsyncLocalStorage.getStore()) !== null && _a !== void 0 ? _a : rootTraceClientContext;
42
- }
43
- function asyncFunctionWrapper(func) {
44
- return (...args) => __awaiter(this, void 0, void 0, function* () {
45
- const parentTraceClientContext = getTraceClientContext();
46
- const traceClientContext = new TraceClientContext();
47
- const lastEntry = parentTraceClientContext.entryStack.at(-1);
48
- if (lastEntry) {
49
- traceClientContext.entryStack.push(lastEntry);
50
- }
51
- const result = yield traceClientContextAsyncLocalStorage
52
- .run(traceClientContext, () => func(...args));
53
- if (traceClientContext.entries.length > parentTraceClientContext.entries.length) {
54
- [traceClientContext.entries, parentTraceClientContext.entries] =
55
- [parentTraceClientContext.entries, traceClientContext.entries];
56
- }
57
- parentTraceClientContext.entries.push(...traceClientContext.entries);
58
- return result;
59
- });
60
- }
61
- // --- API Interaction Client ---
62
- /**
63
- * Client for interacting with Judgment trace API endpoints.
64
- */
65
- class TraceManagerClient {
66
- constructor(apiKey, organizationId) {
67
- if (!apiKey) {
68
- throw new Error("TraceManagerClient requires a Judgment API key.");
69
- }
70
- if (!organizationId) {
71
- throw new Error("TraceManagerClient requires a Judgment Organization ID.");
72
- }
73
- this.apiKey = apiKey;
74
- this.organizationId = organizationId;
75
- }
76
- _fetch(url_1) {
77
- return __awaiter(this, arguments, void 0, function* (url, options = {}) {
78
- const headers = Object.assign({ 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.apiKey}`, 'X-Organization-Id': this.organizationId }, (options.headers || {}));
79
- try {
80
- // Use isomorphic fetch (available globally in modern Node.js and browsers)
81
- const response = yield fetch(url, Object.assign(Object.assign({}, options), { headers: headers }));
82
- // We will return the response object even if !response.ok
83
- // The caller (e.g., saveTrace) is responsible for checking response.ok or response.status
84
- // Handle cases where the response might be empty (e.g., 204 No Content on DELETE)
85
- if (response.status === 204) {
86
- return null; // Indicate success with no content
87
- }
88
- return response;
89
- }
90
- catch (error) {
91
- console.error(`Network or fetch error during ${options.method || 'GET'} ${url}:`, error);
92
- // Re-throw or handle as appropriate for the application context
93
- throw error;
94
- }
95
- });
96
- }
97
- fetchTrace(traceId) {
98
- return __awaiter(this, void 0, void 0, function* () {
99
- return this._fetch(constants_js_1.JUDGMENT_TRACES_FETCH_API_URL, {
100
- method: 'POST',
101
- body: JSON.stringify({ trace_id: traceId }),
102
- });
103
- });
104
- }
105
- saveTrace(traceData) {
106
- return __awaiter(this, void 0, void 0, function* () {
107
- // _fetch now returns the raw response object or throws on network error
108
- const response = yield this._fetch(constants_js_1.JUDGMENT_TRACES_SAVE_API_URL, {
109
- method: 'POST',
110
- body: JSON.stringify(traceData), // Stringify directly here again
111
- });
112
- // Check if _fetch threw a network error (caught below) or returned an invalid object
113
- if (!response) {
114
- // This case should ideally be caught by _fetch's catch block, but double-check
115
- throw new Error('Failed to save trace data: No response received from API.');
116
- }
117
- // Now, check the status code on the received response object
118
- if (response.status === 400) {
119
- // Attempt to get error body for more info
120
- const errorBody = yield response.text();
121
- throw new Error(`Failed to save trace data: Check your Trace name for conflicts, set overwrite=True to overwrite existing traces: ${response.status} ${response.statusText || ''} - ${errorBody}`);
122
- }
123
- else if (!response.ok) { // Handles other errors (5xx, 4xx except 400)
124
- const errorBody = yield response.text();
125
- throw new Error(`Failed to save trace data: Status ${response.status} ${response.statusText || '(No status text)'} - ${errorBody}`);
126
- }
127
- // --- Success Path ---
128
- // Optionally log the UI URL (needs JSON parsing)
129
- let responseData = null;
130
- try {
131
- // Handle 204 No Content specifically
132
- if (response.status === 204) {
133
- responseData = null; // Or maybe { success: true }?
134
- }
135
- else {
136
- responseData = yield response.json(); // Parse JSON only on success
137
- }
138
- }
139
- catch (parseError) {
140
- logger_instance_js_1.default.warn("Failed to parse successful API response JSON.", { error: parseError });
141
- // Depending on requirements, maybe throw, maybe return a default success object
142
- throw new Error(`API request succeeded (${response.status}), but failed to parse JSON response.`);
143
- }
144
- if (responseData === null || responseData === void 0 ? void 0 : responseData.ui_results_url) {
145
- console.info(`
146
- 🔍 View trace: ${responseData.ui_results_url}
147
- `);
148
- }
149
- // Return the parsed data (or null for 204)
150
- return responseData;
151
- });
152
- }
153
- deleteTrace(traceId) {
154
- return __awaiter(this, void 0, void 0, function* () {
155
- // Assuming DELETE method is correct based on REST principles for the delete endpoint
156
- return this._fetch(constants_js_1.JUDGMENT_TRACES_DELETE_API_URL, {
157
- method: 'DELETE',
158
- body: JSON.stringify({ trace_ids: [traceId] }),
159
- });
160
- });
161
- }
162
- deleteTraces(traceIds) {
163
- return __awaiter(this, void 0, void 0, function* () {
164
- return this._fetch(constants_js_1.JUDGMENT_TRACES_DELETE_API_URL, {
165
- method: 'DELETE',
166
- body: JSON.stringify({ trace_ids: traceIds }),
167
- });
168
- });
169
- }
170
- /**
171
- * Calculate token costs directly using the API endpoint.
172
- * This is more accurate than client-side calculation as it uses the most up-to-date pricing.
173
- *
174
- * @param model The model name (e.g. 'gpt-4', 'claude-3-opus-20240229')
175
- * @param promptTokens Number of tokens in the prompt/input
176
- * @param completionTokens Number of tokens in the completion/output
177
- * @returns Object containing token counts and calculated costs in USD
178
- */
179
- calculateTokenCosts(model, promptTokens, completionTokens) {
180
- return __awaiter(this, void 0, void 0, function* () {
181
- try {
182
- // Use the new calculation endpoint
183
- const response = yield this._fetch(constants_js_1.JUDGMENT_CALCULATE_TOKEN_COSTS_API_URL, {
184
- method: 'POST',
185
- body: JSON.stringify({
186
- model,
187
- prompt_tokens: promptTokens,
188
- completion_tokens: completionTokens
189
- })
190
- });
191
- // Check if the response is okay and parse JSON
192
- if (response && response.ok) {
193
- const data = yield response.json();
194
- return data;
195
- }
196
- else if (response) {
197
- // Log error if response was not ok
198
- const errorBody = yield response.text();
199
- logger_instance_js_1.default.warn(`API error calculating token costs for model ${model}: ${response.status} ${response.statusText}`, { errorBody });
200
- return null;
201
- }
202
- else {
203
- // Handle cases where _fetch might return null or undefined (though it shouldn't with current implementation)
204
- logger_instance_js_1.default.warn(`No response received when calculating token costs for model ${model}.`);
205
- return null;
206
- }
207
- }
208
- catch (error) {
209
- logger_instance_js_1.default.warn(`Failed to calculate token costs for model ${model}.`, { error: error instanceof Error ? error.message : String(error) });
210
- return null;
211
- }
212
- });
213
- }
214
- }
215
- exports.TraceManagerClient = TraceManagerClient;
216
- // --- Helper Functions ---
217
- // Helper function to sanitize names (e.g., replace spaces with underscores)
218
- function sanitizeName(name) {
219
- // Replace spaces with underscores and remove potentially problematic characters
220
- // You can adjust the regex further if other characters cause issues.
221
- return name.replace(/\s+/g, '_').replace(/[^a-zA-Z0-9_.-]/g, '');
222
- }
223
- // --- Core Trace Classes ---
224
- /**
225
- * Represents an ongoing trace context.
226
- */
227
- class TraceClient {
228
- constructor(config) {
229
- var _a, _b, _c, _d, _e;
230
- this.traceManager = null; // Made public for wrap access
231
- this._spanDepths = {}; // Track depth of active spans
232
- this.pendingEvaluationRuns = []; // <-- ADDED: Store pending evaluations
233
- this.traceId = config.traceId || (0, uuid_1.v4)();
234
- this.originalName = config.name || 'default_trace'; // Store original
235
- this.name = sanitizeName(this.originalName); // Use sanitized name internally
236
- // If the sanitized name is empty, fallback to a default
237
- if (!this.name) {
238
- console.warn(`Original trace name "${this.originalName}" sanitized to empty string. Using default_trace_${this.traceId.substring(0, 8)}.`);
239
- this.name = `default_trace_${this.traceId.substring(0, 8)}`;
240
- }
241
- this.projectName = (_a = config.projectName) !== null && _a !== void 0 ? _a : config.tracer.projectName;
242
- this.overwrite = (_b = config.overwrite) !== null && _b !== void 0 ? _b : false;
243
- this.rules = (_c = config.rules) !== null && _c !== void 0 ? _c : [];
244
- // Determine effective monitoring status based on tracer and API keys
245
- let effectiveMonitoring = (_d = config.enableMonitoring) !== null && _d !== void 0 ? _d : config.tracer.enableMonitoring;
246
- if (effectiveMonitoring && (!config.apiKey || !config.organizationId)) {
247
- console.warn(`TraceClient ${this.traceId}: Monitoring requires JUDGMENT_API_KEY and JUDGMENT_ORG_ID. Disabling monitoring for this trace.`);
248
- effectiveMonitoring = false;
249
- }
250
- this.enableMonitoring = effectiveMonitoring;
251
- // Evaluations depend on monitoring
252
- this.enableEvaluations = effectiveMonitoring && ((_e = config.enableEvaluations) !== null && _e !== void 0 ? _e : config.tracer.enableEvaluations);
253
- this.parentTraceId = config.parentTraceId;
254
- this.parentName = config.parentName;
255
- this.apiKey = config.apiKey;
256
- this.organizationId = config.organizationId;
257
- this.startTime = Date.now() / 1000;
258
- if (this.enableMonitoring) {
259
- this.traceManager = new TraceManagerClient(this.apiKey, this.organizationId);
260
- }
261
- }
262
- addEntry(entry) {
263
- const traceClientContext = getTraceClientContext();
264
- if (this.enableMonitoring) {
265
- traceClientContext.entries.push(entry);
266
- }
267
- }
268
- recordInput(inputs) {
269
- const traceClientContext = getTraceClientContext();
270
- const currentEntry = traceClientContext.entryStack.at(-1);
271
- if (!currentEntry || !currentEntry.span_id) {
272
- console.warn(`No current entry to record input to\nStack trace: ${new Error().stack}`);
273
- return;
274
- }
275
- this.addEntry({
276
- type: 'input',
277
- span_id: currentEntry.span_id,
278
- inputs,
279
- function: currentEntry.function,
280
- depth: this._spanDepths[currentEntry.span_id],
281
- created_at: Date.now() / 1000,
282
- span_type: currentEntry.span_type,
283
- message: `Inputs to ${currentEntry.function}`
284
- });
285
- }
286
- recordOutput(output) {
287
- const traceClientContext = getTraceClientContext();
288
- const currentEntry = traceClientContext.entryStack.at(-1);
289
- if (!currentEntry || !currentEntry.span_id) {
290
- console.warn(`No current entry to record output to\nStack trace: ${new Error().stack}`);
291
- return;
292
- }
293
- this.addEntry({
294
- type: 'output',
295
- span_id: currentEntry.span_id,
296
- output,
297
- function: currentEntry.function,
298
- depth: this._spanDepths[currentEntry.span_id],
299
- created_at: Date.now() / 1000,
300
- span_type: currentEntry.span_type,
301
- message: `Output from ${currentEntry.function}`
302
- });
303
- }
304
- recordError(error) {
305
- const traceClientContext = getTraceClientContext();
306
- const currentEntry = traceClientContext.entryStack.at(-1);
307
- if (!currentEntry || !currentEntry.span_id) {
308
- console.warn(`No current entry to record error to\nStack trace: ${new Error().stack}`);
309
- return;
310
- }
311
- this.addEntry({
312
- type: 'error',
313
- span_id: currentEntry.span_id,
314
- output: error,
315
- function: currentEntry.function,
316
- depth: this._spanDepths[currentEntry.span_id],
317
- created_at: Date.now() / 1000,
318
- span_type: currentEntry.span_type,
319
- message: `Error from ${currentEntry.function}`
320
- });
321
- }
322
- startSpan(name, options = {}) {
323
- var _a;
324
- const traceClientContext = getTraceClientContext();
325
- const parentEntry = traceClientContext.entryStack.at(-1);
326
- const spanId = (0, uuid_1.v4)();
327
- const spanType = (_a = options.spanType) !== null && _a !== void 0 ? _a : 'span';
328
- const startTime = Date.now() / 1000;
329
- let depth = 0, parentSpanId = undefined;
330
- if (parentEntry && parentEntry.span_id) {
331
- depth = this._spanDepths[parentEntry.span_id] + 1;
332
- parentSpanId = parentEntry.span_id;
333
- }
334
- this._spanDepths[spanId] = depth;
335
- const entry = {
336
- type: 'enter',
337
- function: name,
338
- span_id: spanId,
339
- depth: depth,
340
- created_at: startTime,
341
- span_type: spanType,
342
- parent_span_id: parentSpanId,
343
- message: name
344
- };
345
- this.addEntry(entry);
346
- traceClientContext.entryStack.push(entry);
347
- }
348
- endSpan() {
349
- const traceClientContext = getTraceClientContext();
350
- const enterEntry = traceClientContext.entryStack.pop();
351
- if (!enterEntry || !enterEntry.span_id) {
352
- console.warn("No enter entry to end");
353
- return;
354
- }
355
- const endTime = Date.now() / 1000;
356
- const duration = endTime - enterEntry.created_at;
357
- this.addEntry({
358
- type: 'exit',
359
- function: enterEntry.function,
360
- span_id: enterEntry.span_id,
361
- depth: this._spanDepths[enterEntry.span_id],
362
- created_at: endTime,
363
- duration: duration,
364
- span_type: enterEntry.span_type,
365
- message: `← ${enterEntry.function}`
366
- });
367
- // Clean up depth tracking
368
- delete this._spanDepths[enterEntry.span_id];
369
- }
370
- *span(name, options = {}) {
371
- if (!this.enableMonitoring) {
372
- yield this;
373
- }
374
- else {
375
- this.startSpan(name, options);
376
- yield this;
377
- this.endSpan();
378
- }
379
- }
380
- /**
381
- * Retrieves the ID of the currently active span in this trace context.
382
- * Relies on AsyncLocalStorage context established by observe/span.
383
- * @returns {string | undefined} The ID of the current span, or undefined if none is active.
384
- */
385
- getCurrentSpanId() {
386
- const traceClientContext = getTraceClientContext(); // Internal function using AsyncLocalStorage
387
- if (!traceClientContext) {
388
- // Should ideally not happen if called within an observe/trace context
389
- console.warn("[Judgeval] getCurrentSpanId called outside of an active trace context.");
390
- return undefined;
391
- }
392
- const currentEntry = traceClientContext.entryStack.at(-1); // Get the latest 'enter' entry
393
- if (!currentEntry) {
394
- // This might happen if called right at the start of a trace before the first span
395
- // console.warn("[Judgeval] getCurrentSpanId called but span stack is empty.");
396
- return undefined;
397
- }
398
- return currentEntry.span_id;
399
- }
400
- getDuration() {
401
- return (Date.now() / 1000) - this.startTime;
402
- }
403
- condenseTrace(rawEntries) {
404
- var _a, _b, _c, _d, _e;
405
- const spansById = {};
406
- for (const entry of rawEntries) {
407
- const spanId = entry.span_id;
408
- if (!spanId)
409
- continue;
410
- if (!spansById[spanId]) {
411
- spansById[spanId] = {
412
- span_id: spanId,
413
- function: entry.function || 'unknown',
414
- depth: (_a = entry.depth) !== null && _a !== void 0 ? _a : 0,
415
- created_at: new Date(((_b = entry.created_at) !== null && _b !== void 0 ? _b : 0) * 1000).toISOString(), // Convert number to ISO string
416
- trace_id: this.traceId, // Add trace_id
417
- parent_span_id: entry.parent_span_id,
418
- span_type: entry.span_type || 'span',
419
- inputs: null,
420
- output: null,
421
- duration: null,
422
- children: []
423
- };
424
- }
425
- const currentSpanData = spansById[spanId];
426
- switch (entry.type) {
427
- case 'enter':
428
- currentSpanData.function = entry.function || currentSpanData.function;
429
- currentSpanData.depth = (_c = entry.depth) !== null && _c !== void 0 ? _c : currentSpanData.depth;
430
- currentSpanData.created_at = new Date(((_d = entry.created_at) !== null && _d !== void 0 ? _d : 0) * 1000).toISOString(); // Ensure created_at is string on update
431
- currentSpanData.parent_span_id = entry.parent_span_id;
432
- currentSpanData.span_type = entry.span_type || currentSpanData.span_type;
433
- currentSpanData.start_time = entry.created_at; // Keep original number for duration calc
434
- break;
435
- case 'exit':
436
- currentSpanData.duration = (_e = entry.duration) !== null && _e !== void 0 ? _e : currentSpanData.duration;
437
- currentSpanData.end_time = entry.created_at; // Keep original number for duration calc
438
- if (currentSpanData.duration === null && currentSpanData.start_time && currentSpanData.end_time) {
439
- currentSpanData.duration = currentSpanData.end_time - currentSpanData.start_time;
440
- }
441
- break;
442
- case 'input':
443
- if (currentSpanData.inputs === null && entry.inputs) {
444
- currentSpanData.inputs = entry.inputs;
445
- }
446
- else if (typeof currentSpanData.inputs === 'object' && typeof entry.inputs === 'object') {
447
- currentSpanData.inputs = Object.assign(Object.assign({}, currentSpanData.inputs), entry.inputs);
448
- }
449
- break;
450
- case 'output':
451
- case 'error':
452
- currentSpanData.output = entry.output;
453
- break;
454
- }
455
- }
456
- const spansList = Object.values(spansById).map(span => {
457
- if (span.duration === null && span.start_time && span.end_time) {
458
- span.duration = span.end_time - span.start_time;
459
- }
460
- delete span.start_time;
461
- delete span.end_time;
462
- return span;
463
- });
464
- const childrenMap = {};
465
- const roots = [];
466
- const spanMap = {};
467
- const sortedCondensedList = [];
468
- const visited = new Set();
469
- for (const span of spansList) {
470
- spanMap[span.span_id] = span;
471
- const parentId = span.parent_span_id;
472
- if (parentId === undefined || parentId === null) {
473
- roots.push(span);
474
- }
475
- else {
476
- if (!childrenMap[parentId]) {
477
- childrenMap[parentId] = [];
478
- }
479
- childrenMap[parentId].push(span);
480
- }
481
- }
482
- // Sort using parsed dates
483
- roots.sort((a, b) => Date.parse(a.created_at) - Date.parse(b.created_at));
484
- for (const parentId in childrenMap) {
485
- // Sort using parsed dates
486
- childrenMap[parentId].sort((a, b) => Date.parse(a.created_at) - Date.parse(b.created_at));
487
- }
488
- function buildFlatListDfs(span) {
489
- if (visited.has(span.span_id))
490
- return;
491
- visited.add(span.span_id);
492
- sortedCondensedList.push(span);
493
- const children = childrenMap[span.span_id] || [];
494
- for (const child of children) {
495
- buildFlatListDfs(child);
496
- }
497
- }
498
- for (const rootSpan of roots) {
499
- buildFlatListDfs(rootSpan);
500
- }
501
- for (const span of spansList) {
502
- if (!visited.has(span.span_id)) {
503
- console.warn(`Orphaned span detected: ${span.span_id}, adding to end of list.`);
504
- buildFlatListDfs(span);
505
- }
506
- }
507
- return sortedCondensedList;
508
- }
509
- save() {
510
- return __awaiter(this, arguments, void 0, function* (emptySave = false) {
511
- var _a;
512
- // If monitoring is disabled or trace hasn't started, don't save
513
- if (!this.enableMonitoring || this.startTime === -1 || !this.traceManager) {
514
- logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Monitoring disabled or trace not started. Not saving.`);
515
- return null;
516
- }
517
- const endTime = Date.now() / 1000; // Current time in seconds
518
- const duration = endTime - this.startTime;
519
- const condensedEntries = this.condenseTrace(getTraceClientContext().entries);
520
- // Calculate token counts and costs
521
- const tokenCounts = {
522
- prompt_tokens: 0,
523
- completion_tokens: 0,
524
- total_tokens: 0,
525
- prompt_tokens_cost_usd: 0.0,
526
- completion_tokens_cost_usd: 0.0,
527
- total_cost_usd: 0.0
528
- };
529
- // Use a Set to avoid double-counting tokens from nested calls if structure is complex
530
- // Note: Assuming span_ids are unique across the trace
531
- const processedSpanIds = new Set();
532
- for (const entry of condensedEntries) {
533
- if (processedSpanIds.has(entry.span_id))
534
- continue;
535
- processedSpanIds.add(entry.span_id);
536
- if (entry.span_type === 'llm' && ((_a = entry.output) === null || _a === void 0 ? void 0 : _a.usage)) {
537
- const usage = entry.output.usage;
538
- // Sum tokens
539
- tokenCounts.prompt_tokens += usage.prompt_tokens || usage.input_tokens || 0;
540
- tokenCounts.completion_tokens += usage.completion_tokens || usage.output_tokens || 0;
541
- tokenCounts.total_tokens += usage.total_tokens ||
542
- ((usage.prompt_tokens || usage.input_tokens || 0) +
543
- (usage.completion_tokens || usage.output_tokens || 0));
544
- // Sum costs embedded in the usage object (if they exist)
545
- if (usage.prompt_tokens_cost_usd !== undefined) {
546
- tokenCounts.prompt_tokens_cost_usd += usage.prompt_tokens_cost_usd;
547
- }
548
- if (usage.completion_tokens_cost_usd !== undefined) {
549
- tokenCounts.completion_tokens_cost_usd += usage.completion_tokens_cost_usd;
550
- }
551
- if (usage.total_cost_usd !== undefined) {
552
- tokenCounts.total_cost_usd += usage.total_cost_usd;
553
- }
554
- }
555
- }
556
- // --- Retrieve and include pending evaluation runs ---
557
- const evaluationRunsToSave = [...this.pendingEvaluationRuns];
558
- // ----------------------------------------------------
559
- const traceData = {
560
- trace_id: this.traceId,
561
- name: this.name, // Use the sanitized name
562
- project_name: this.projectName,
563
- created_at: new Date(this.startTime * 1000).toISOString(), // Convert start time to ISO string
564
- duration: duration < 0 ? 0 : duration,
565
- token_counts: tokenCounts,
566
- entries: condensedEntries, // Send the potentially nested structure from condenseTrace
567
- evaluation_runs: evaluationRunsToSave, // <-- ADDED: Include collected evaluations
568
- overwrite: this.overwrite,
569
- parent_trace_id: this.parentTraceId,
570
- parent_name: this.parentName,
571
- };
572
- // <<< ADD LOGGING HERE >>>
573
- logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Payload to be saved:`, JSON.stringify(traceData, null, 2));
574
- // <<< ADD SPECIFIC LOGGING FOR EVALUATION RUNS (using INFO level) >>>
575
- logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Evaluation runs included in payload:`, JSON.stringify(traceData.evaluation_runs, null, 2));
576
- // <<< END SPECIFIC LOGGING >>>
577
- if (emptySave) {
578
- // Skip actual saving if emptySave is true (used for context management in generators)
579
- logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] emptySave=true, skipping actual save call.`);
580
- return { traceId: this.traceId, traceData };
581
- }
582
- try {
583
- logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Calling traceManager.saveTrace...`);
584
- const response = yield this.traceManager.saveTrace(traceData);
585
- logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Trace saved successfully. Response:`, response);
586
- // Reset trace context after successful save
587
- getTraceClientContext().entries = [];
588
- getTraceClientContext().entryStack = [];
589
- this.pendingEvaluationRuns = []; // <-- ADDED: Clear pending evaluations
590
- this.startTime = -1; // Reset start time
591
- return { traceId: this.traceId, traceData: traceData }; // Return payload on success
592
- }
593
- catch (error) {
594
- logger_instance_js_1.default.error(`[TraceClient ${this.traceId}] Error saving trace:`, error);
595
- // Optionally reset context even on error?
596
- // getTraceClientContext().entries = [];
597
- // getTraceClientContext().entryStack = [];
598
- // this.startTime = -1;
599
- return null; // Indicate save failure
600
- }
601
- });
602
- }
603
- print() {
604
- if (!this.enableMonitoring) {
605
- // Keep console.log for direct user output when print() is called
606
- console.log("Monitoring was disabled. No trace entries recorded.");
607
- return;
608
- }
609
- const traceClientContext = getTraceClientContext();
610
- if (traceClientContext.entries.length === 0) {
611
- // Keep console.log for direct user output when print() is called
612
- console.log("No trace entries recorded.");
613
- return;
614
- }
615
- // Keep console.log for direct user output when print() is called
616
- console.log(`\n--- Trace Details: ${this.name} (ID: ${this.traceId}) ---`);
617
- traceClientContext.entries.forEach(entry => {
618
- var _a;
619
- const indent = " ".repeat((_a = entry.depth) !== null && _a !== void 0 ? _a : 0);
620
- const timeStr = entry.created_at ? `@ ${new Date(entry.created_at * 1000).toISOString()}` : '';
621
- const shortSpanId = entry.span_id ? `(id: ${entry.span_id.substring(0, 8)}...)` : '';
622
- const shortParentId = entry.parent_span_id ? `(parent: ${entry.parent_span_id.substring(0, 8)}...)` : '';
623
- try {
624
- switch (entry.type) {
625
- case 'enter':
626
- console.log(`${indent}→ ${entry.function || 'unknown'} ${shortSpanId} ${shortParentId} [${entry.span_type || 'span'}] ${timeStr}`);
627
- break;
628
- case 'exit':
629
- const durationStr = entry.duration !== undefined ? `(${entry.duration.toFixed(3)}s)` : '';
630
- // Keep console.log
631
- console.log(`${indent}← ${entry.function || 'unknown'} ${shortSpanId} ${durationStr} ${timeStr}`);
632
- break;
633
- case 'input':
634
- let inputStr = JSON.stringify(entry.inputs);
635
- if (inputStr && inputStr.length > 200) {
636
- inputStr = inputStr.substring(0, 197) + '...';
637
- }
638
- // Keep console.log
639
- console.log(`${indent} Input (for ${shortSpanId}): ${inputStr || '{}'}`);
640
- break;
641
- case 'output':
642
- case 'error':
643
- let outputStr = JSON.stringify(entry.output);
644
- if (outputStr && outputStr.length > 200) {
645
- outputStr = outputStr.substring(0, 197) + '...';
646
- }
647
- const prefix = entry.type === 'error' ? 'Error' : 'Output';
648
- // Keep console.log
649
- console.log(`${indent} ${prefix} (for ${shortSpanId}): ${outputStr || 'null'}`);
650
- break;
651
- }
652
- }
653
- catch (stringifyError) {
654
- const errorMessage = stringifyError instanceof Error ? stringifyError.message : String(stringifyError);
655
- // Keep console.log
656
- console.log(`${indent}! Error formatting entry: ${errorMessage}`);
657
- console.log(`${indent} Raw entry:`, entry);
658
- }
659
- });
660
- // Keep console.log
661
- console.log(`--- End Trace: ${this.name} ---`);
662
- }
663
- delete() {
664
- return __awaiter(this, void 0, void 0, function* () {
665
- if (!this.enableMonitoring || !this.traceManager) {
666
- logger_instance_js_1.default.warn(`Cannot delete trace ${this.traceId}, monitoring disabled or manager missing.`);
667
- return null;
668
- }
669
- try {
670
- const result = yield this.traceManager.deleteTrace(this.traceId);
671
- logger_instance_js_1.default.info(`Trace ${this.traceId} deleted successfully.`);
672
- return result;
673
- }
674
- catch (error) {
675
- logger_instance_js_1.default.error(`Failed to delete trace ${this.traceId}.`, { error: error instanceof Error ? error.message : String(error) });
676
- throw error; // Re-throw after logging
677
- }
678
- });
679
- }
680
- /**
681
- * Asynchronously evaluate an example using the provided scorers,
682
- * embedding the evaluation request into the trace data.
683
- * Ported from the Python SDK's async_evaluate method.
684
- *
685
- * @param scorers Array of scorers to use for evaluation
686
- * @param options Evaluation options including input, outputs, and metadata
687
- * @returns Promise that resolves when the evaluation entry has been added to the trace
688
- */
689
- asyncEvaluate(scorers_1) {
690
- return __awaiter(this, arguments, void 0, function* (scorers, options = {}) {
691
- if (!this.enableEvaluations) {
692
- logger_instance_js_1.default.warn("Evaluations are disabled. Skipping async evaluation.");
693
- return;
694
- }
695
- if (!scorers || scorers.length === 0) {
696
- logger_instance_js_1.default.warn("No scorers provided. Skipping async evaluation.");
697
- return;
698
- }
699
- // Filter for APIJudgmentScorers as the backend evaluation needs them
700
- const apiScorers = scorers.filter((s) => s instanceof base_scorer_js_1.APIJudgmentScorer);
701
- if (apiScorers.length === 0) {
702
- logger_instance_js_1.default.warn("No APIJudgmentScorers found in the provided scorers list. Skipping async evaluation as backend requires API scorers.");
703
- return;
704
- }
705
- // Process rules (currently just using this.rules directly)
706
- const loadedRules = this.rules; // TODO: Add ScorerWrapper-like processing if needed in TS
707
- // Prevent using JudgevalScorer with rules - only APIJudgmentScorer allowed with rules
708
- if (loadedRules && loadedRules.length > 0 && scorers.some(s => !(s instanceof base_scorer_js_1.APIJudgmentScorer))) {
709
- throw new Error("Cannot use Judgeval scorers, you can only use API scorers when using rules. Please either remove rules or use only APIJudgmentScorer types.");
710
- }
711
- // Create example structure matching Python/backend expectations
712
- const example = {
713
- input: options.input || "",
714
- actual_output: options.actualOutput || "",
715
- expected_output: options.expectedOutput || "",
716
- context: options.context || [],
717
- retrieval_context: options.retrievalContext || [],
718
- tools_called: options.toolsCalled || [],
719
- expected_tools: options.expectedTools || [],
720
- additional_metadata: options.additionalMetadata || {},
721
- trace_id: this.traceId
722
- };
723
- try {
724
- const traceClientContext = getTraceClientContext();
725
- const currentEntry = traceClientContext.entryStack.at(-1);
726
- if (!currentEntry) {
727
- logger_instance_js_1.default.warn(`No current entry to associate evaluation with\\nStack trace: ${new Error().stack}`);
728
- return;
729
- }
730
- const currentSpanId = currentEntry.span_id; // Get the span ID
731
- // --- Create evaluation run name (similar to Python) ---
732
- // Capitalize scorer names
733
- const scorerNames = apiScorers.map(scorer => {
734
- var _a;
735
- // Attempt to get score_type, fallback to class name or Unknown
736
- const name = (scorer === null || scorer === void 0 ? void 0 : scorer.scoreType) || ((_a = scorer === null || scorer === void 0 ? void 0 : scorer.constructor) === null || _a === void 0 ? void 0 : _a.name) || "Unknown";
737
- return name.charAt(0).toUpperCase() + name.slice(1);
738
- }).join(',');
739
- // Use trace name and shortened span ID (or trace ID if no span)
740
- const idPart = currentSpanId ? currentSpanId.substring(0, 8) : this.traceId.substring(0, 8);
741
- const evalName = `${this.name.charAt(0).toUpperCase() + this.name.slice(1)}-${idPart}-[${scorerNames}]`;
742
- // --- End eval name creation ---
743
- // Construct the evaluation payload
744
- const evalRunPayload = {
745
- organization_id: this.organizationId,
746
- log_results: options.logResults !== false, // Default to true
747
- project_name: this.projectName,
748
- eval_name: evalName,
749
- examples: [example],
750
- scorers: apiScorers, // Use the filtered list of API scorers
751
- model: options.model || "",
752
- metadata: {}, // Matches Python tracer
753
- judgment_api_key: this.apiKey,
754
- override: this.overwrite, // Use trace's overwrite setting
755
- rules: loadedRules, // Pass the processed rules
756
- trace_span_id: currentSpanId // <<< RENAMED: Assign the current span ID (matching backend)
757
- };
758
- // --- Log the payload before storing (using INFO level) ---
759
- logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Storing EvaluationRunPayload for span ${currentSpanId}:`, JSON.stringify(evalRunPayload, null, 2));
760
- // --- End log before store ---
761
- // --- Store the payload instead of sending ---
762
- this.pendingEvaluationRuns.push(evalRunPayload);
763
- logger_instance_js_1.default.info(`[TraceClient ${this.traceId}] Evaluation payload stored for span ${currentSpanId}`);
764
- // --- End store payload ---
765
- }
766
- catch (error) {
767
- logger_instance_js_1.default.error(`Failed during asyncEvaluate execution: ${error instanceof Error ? error.message : String(error)}`);
768
- throw error; // Re-throw after logging
769
- }
770
- });
771
- }
772
- // OPTIONAL: Add a method to get the original name if needed elsewhere
773
- getOriginalName() {
774
- return this.originalName;
775
- }
776
- }
777
- exports.TraceClient = TraceClient;
778
- /**
779
- * Singleton Tracer class. Manages overall tracing configuration and trace creation.
780
- */
781
- class Tracer {
782
- constructor(config) {
783
- var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p;
784
- this.initialized = false;
785
- const isNode = typeof process !== 'undefined' && process.versions != null && process.versions.node != null;
786
- const envApiKey = isNode ? (_a = process.env) === null || _a === void 0 ? void 0 : _a.JUDGMENT_API_KEY : undefined;
787
- const envOrgId = isNode ? (_b = process.env) === null || _b === void 0 ? void 0 : _b.JUDGMENT_ORG_ID : undefined;
788
- const envProjectName = isNode ? (_c = process.env) === null || _c === void 0 ? void 0 : _c.JUDGMENT_PROJECT_NAME : undefined;
789
- const envMonitoring = isNode ? (_d = process.env) === null || _d === void 0 ? void 0 : _d.JUDGMENT_MONITORING : 'true';
790
- const envEvaluations = isNode ? (_e = process.env) === null || _e === void 0 ? void 0 : _e.JUDGMENT_EVALUATIONS : 'true';
791
- this.apiKey = (_g = (_f = config === null || config === void 0 ? void 0 : config.apiKey) !== null && _f !== void 0 ? _f : envApiKey) !== null && _g !== void 0 ? _g : '';
792
- this.organizationId = (_j = (_h = config === null || config === void 0 ? void 0 : config.organizationId) !== null && _h !== void 0 ? _h : envOrgId) !== null && _j !== void 0 ? _j : '';
793
- this.projectName = (_l = (_k = config === null || config === void 0 ? void 0 : config.projectName) !== null && _k !== void 0 ? _k : envProjectName) !== null && _l !== void 0 ? _l : 'default_project';
794
- this.defaultRules = (_m = config === null || config === void 0 ? void 0 : config.rules) !== null && _m !== void 0 ? _m : [];
795
- let effectiveMonitoring = (_o = config === null || config === void 0 ? void 0 : config.enableMonitoring) !== null && _o !== void 0 ? _o : ((envMonitoring === null || envMonitoring === void 0 ? void 0 : envMonitoring.toLowerCase()) !== 'false');
796
- if (effectiveMonitoring && (!this.apiKey || !this.organizationId)) {
797
- console.warn("JUDGMENT_API_KEY or JUDGMENT_ORG_ID missing. Monitoring disabled.");
798
- effectiveMonitoring = false;
799
- }
800
- this.enableMonitoring = effectiveMonitoring;
801
- this.enableEvaluations = effectiveMonitoring && ((_p = config === null || config === void 0 ? void 0 : config.enableEvaluations) !== null && _p !== void 0 ? _p : ((envEvaluations === null || envEvaluations === void 0 ? void 0 : envEvaluations.toLowerCase()) !== 'false'));
802
- this.initialized = true;
803
- }
804
- static getInstance(config) {
805
- if (!Tracer.instance) {
806
- Tracer.instance = new Tracer(config);
807
- }
808
- else if (config && !Tracer.instance.initialized) {
809
- console.warn("Tracer getInstance called with config after implicit initialization. Re-initializing.");
810
- Tracer.instance = new Tracer(config);
811
- }
812
- else if (config && Tracer.instance.initialized) {
813
- if (config.projectName && config.projectName !== Tracer.instance.projectName) {
814
- console.warn(`Attempting to re-initialize Tracer with different project_name. Original will be used: '${Tracer.instance.projectName}'.`);
815
- }
816
- if (config.rules && config.rules.length > 0 && Tracer.instance.defaultRules.length === 0) {
817
- try {
818
- console.warn("Setting default rules on Tracer instance after initial creation.");
819
- Tracer.instance.defaultRules = config.rules;
820
- }
821
- catch (e) {
822
- console.error("Failed to set default rules after tracer initialization:", e);
823
- }
824
- }
825
- else if (config.rules && JSON.stringify(config.rules) !== JSON.stringify(Tracer.instance.defaultRules)) {
826
- console.warn("Attempting to change default rules on Tracer after initialization. Original rules will be used.");
827
- }
828
- }
829
- return Tracer.instance;
830
- }
831
- getCurrentTrace() {
832
- return this.currentTrace;
833
- }
834
- startTrace(name, config) {
835
- var _a, _b;
836
- const parentTrace = this.getCurrentTrace();
837
- const projectName = (_a = config.projectName) !== null && _a !== void 0 ? _a : this.projectName;
838
- const effectiveRules = Object.values(Object.fromEntries([...this.defaultRules, ...((_b = config.rules) !== null && _b !== void 0 ? _b : [])]
839
- .map(rule => { var _a; return [(_a = rule.rule_id) !== null && _a !== void 0 ? _a : rule.name, rule]; })));
840
- const traceClient = new TraceClient({
841
- tracer: this,
842
- name: name,
843
- projectName: projectName,
844
- overwrite: config.overwrite,
845
- rules: effectiveRules,
846
- enableMonitoring: this.enableMonitoring,
847
- enableEvaluations: this.enableEvaluations,
848
- parentTraceId: parentTrace === null || parentTrace === void 0 ? void 0 : parentTrace.traceId,
849
- parentName: parentTrace === null || parentTrace === void 0 ? void 0 : parentTrace.name,
850
- apiKey: this.apiKey,
851
- organizationId: this.organizationId,
852
- });
853
- return traceClient;
854
- }
855
- *trace(name, options = {}) {
856
- var _a;
857
- const trace = this.startTrace(name, Object.assign({}, options));
858
- const shouldCreateRootSpan = (_a = options.createRootSpan) !== null && _a !== void 0 ? _a : true;
859
- const prevTrace = this.currentTrace;
860
- if (shouldCreateRootSpan) {
861
- for (const span of trace.span(name, { spanType: 'chain' })) {
862
- this.currentTrace = trace;
863
- yield trace;
864
- this.currentTrace = prevTrace;
865
- }
866
- }
867
- else {
868
- this.currentTrace = trace;
869
- yield trace;
870
- this.currentTrace = prevTrace;
871
- }
872
- if (trace.enableMonitoring) {
873
- trace.save(false).catch(saveErr => {
874
- console.error(`Failed to save completed trace '${name}' (${trace.traceId}):`, saveErr);
875
- });
876
- }
877
- }
878
- observe(options) {
879
- if (!this.enableMonitoring) {
880
- return (func) => (...args) => Promise.resolve(func(...args));
881
- }
882
- return (func) => {
883
- var _a, _b, _c;
884
- const spanName = (_b = (_a = options === null || options === void 0 ? void 0 : options.name) !== null && _a !== void 0 ? _a : func.name) !== null && _b !== void 0 ? _b : 'anonymous_function';
885
- const spanType = (_c = options === null || options === void 0 ? void 0 : options.spanType) !== null && _c !== void 0 ? _c : 'span';
886
- return asyncFunctionWrapper(((...args) => __awaiter(this, void 0, void 0, function* () {
887
- const currentTrace = this.getCurrentTrace();
888
- let output;
889
- let error;
890
- if (!currentTrace) {
891
- for (const trace of this.trace(spanName, { createRootSpan: false })) {
892
- for (const span of trace.span(spanName, { spanType })) {
893
- span.recordInput({ args: args });
894
- try {
895
- span.recordOutput(output = yield func(...args));
896
- }
897
- catch (e) {
898
- span.recordError(error = e);
899
- }
900
- }
901
- }
902
- }
903
- else {
904
- for (const span of currentTrace.span(spanName, { spanType })) {
905
- span.recordInput({ args: args });
906
- try {
907
- span.recordOutput(output = yield func(...args));
908
- }
909
- catch (e) {
910
- span.recordError(error = e);
911
- }
912
- }
913
- }
914
- if (error) {
915
- throw error;
916
- }
917
- // @ts-expect-error output is assigned
918
- return output;
919
- })).bind(func));
920
- };
921
- }
922
- }
923
- exports.Tracer = Tracer;
924
- Tracer.instance = null;
925
- // --- Helper Functions for Wrapping LLM Clients --- //
926
- // Return owner and method name for explicit patching
927
- function _getClientConfig(client) {
928
- var _a, _b, _c, _d, _e, _f;
929
- // Check OpenAI structure first
930
- if (client instanceof openai_1.default && typeof ((_b = (_a = client === null || client === void 0 ? void 0 : client.chat) === null || _a === void 0 ? void 0 : _a.completions) === null || _b === void 0 ? void 0 : _b.create) === 'function') {
931
- return {
932
- spanName: "OPENAI_API_CALL",
933
- originalMethod: client.chat.completions.create,
934
- methodOwner: client.chat.completions,
935
- methodName: 'create'
936
- };
937
- }
938
- // Check Anthropic structure next
939
- else if (client instanceof sdk_1.default && typeof ((_c = client === null || client === void 0 ? void 0 : client.messages) === null || _c === void 0 ? void 0 : _c.create) === 'function') {
940
- return {
941
- spanName: "ANTHROPIC_API_CALL",
942
- originalMethod: client.messages.create,
943
- methodOwner: client.messages,
944
- methodName: 'create'
945
- };
946
- }
947
- // Check for Together structure (duck typing - looking for .chat.completions.create for v0.7.0)
948
- else if (typeof ((_e = (_d = client === null || client === void 0 ? void 0 : client.chat) === null || _d === void 0 ? void 0 : _d.completions) === null || _e === void 0 ? void 0 : _e.create) === 'function') {
949
- const chatCompletionsObj = client.chat.completions;
950
- return {
951
- spanName: "TOGETHER_API_CALL",
952
- originalMethod: chatCompletionsObj.create,
953
- methodOwner: chatCompletionsObj, // Owner is chat.completions
954
- methodName: 'create'
955
- };
956
- }
957
- // Fallback/Warning if none match
958
- logger_instance_js_1.default.warn("Cannot wrap client: Unsupported type or incompatible SDK structure.", { clientType: (_f = client === null || client === void 0 ? void 0 : client.constructor) === null || _f === void 0 ? void 0 : _f.name });
959
- return null;
960
- }
961
- function _formatInputData(client, args) {
962
- var _a, _b;
963
- const params = args[0] || {};
964
- try {
965
- // Check for OpenAI or Together (assuming chat structure)
966
- if (client instanceof openai_1.default || (((_a = client === null || client === void 0 ? void 0 : client.constructor) === null || _a === void 0 ? void 0 : _a.name) === 'Together' && ((_b = client === null || client === void 0 ? void 0 : client.chat) === null || _b === void 0 ? void 0 : _b.completions))) {
967
- return { model: params.model, messages: params.messages, /* other potential params */ };
968
- }
969
- else if (client instanceof sdk_1.default) {
970
- return { model: params.model, messages: params.messages, max_tokens: params.max_tokens, };
971
- }
972
- }
973
- catch (e) {
974
- logger_instance_js_1.default.error("Error formatting LLM input:", { error: e instanceof Error ? e.message : String(e), params });
975
- return { raw_params: params };
976
- }
977
- return { raw_params: params };
978
- }
979
- function _formatOutputData(client, response) {
980
- var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
981
- try {
982
- // Separate handling for OpenAI and Together, assuming Together might differ
983
- if (client instanceof openai_1.default && ((_b = (_a = response === null || response === void 0 ? void 0 : response.choices) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.message)) {
984
- const message = response.choices[0].message;
985
- return { content: message.content, usage: response.usage, };
986
- }
987
- else if ( // Check for Together (assuming chat structure)
988
- (((_c = client === null || client === void 0 ? void 0 : client.constructor) === null || _c === void 0 ? void 0 : _c.name) === 'Together' && ((_d = client === null || client === void 0 ? void 0 : client.chat) === null || _d === void 0 ? void 0 : _d.completions)) &&
989
- ((_f = (_e = response === null || response === void 0 ? void 0 : response.choices) === null || _e === void 0 ? void 0 : _e[0]) === null || _f === void 0 ? void 0 : _f.message)) {
990
- const message = response.choices[0].message;
991
- // Together API might return usage directly or nested, check defensively
992
- const usage = (response === null || response === void 0 ? void 0 : response.usage) || ((_h = (_g = response === null || response === void 0 ? void 0 : response.choices) === null || _g === void 0 ? void 0 : _g[0]) === null || _h === void 0 ? void 0 : _h.usage) || null;
993
- return { content: message === null || message === void 0 ? void 0 : message.content, usage: usage, };
994
- }
995
- else if (client instanceof sdk_1.default && ((_j = response === null || response === void 0 ? void 0 : response.content) === null || _j === void 0 ? void 0 : _j[0])) {
996
- const textContent = response.content.filter((block) => block.type === 'text').map((block) => block.text).join('');
997
- // Reconstruct usage for Anthropic if needed
998
- const usage = {
999
- input_tokens: (_k = response.usage) === null || _k === void 0 ? void 0 : _k.input_tokens,
1000
- output_tokens: (_l = response.usage) === null || _l === void 0 ? void 0 : _l.output_tokens,
1001
- total_tokens: (((_m = response.usage) === null || _m === void 0 ? void 0 : _m.input_tokens) || 0) + (((_o = response.usage) === null || _o === void 0 ? void 0 : _o.output_tokens) || 0)
1002
- };
1003
- return { content: textContent, usage: usage, };
1004
- }
1005
- }
1006
- catch (e) {
1007
- logger_instance_js_1.default.error("Error formatting LLM output:", { error: e instanceof Error ? e.message : String(e) });
1008
- return { formatting_error: String(e), raw_response: response };
1009
- }
1010
- // Return raw if structure doesn't match known patterns
1011
- return { raw_response: response };
1012
- }
1013
- // --- The Wrap Function --- //
1014
- function wrap(client) {
1015
- var _a, _b, _c;
1016
- const tracer = Tracer.getInstance();
1017
- if (!tracer.enableMonitoring) {
1018
- logger_instance_js_1.default.info("Global monitoring disabled, client wrapping skipped.");
1019
- return client;
1020
- }
1021
- // Generic logic for all clients (OpenAI, Anthropic, Together via duck-typing)
1022
- const config = _getClientConfig(client);
1023
- if (!config) {
1024
- return client; // Warning already logged
1025
- }
1026
- const { spanName, originalMethod, methodOwner, methodName } = config;
1027
- const boundOriginalMethod = originalMethod.bind(methodOwner);
1028
- const tracedMethod = (...args) => __awaiter(this, void 0, void 0, function* () {
1029
- var _a, _b, _c, _d, _e;
1030
- const currentTrace = tracer.getCurrentTrace();
1031
- if (!currentTrace || !currentTrace.enableMonitoring) {
1032
- return boundOriginalMethod(...args);
1033
- }
1034
- let response;
1035
- for (const span of currentTrace.span(spanName, { spanType: 'llm' })) {
1036
- // 1. Record Input (extract data needed for recording)
1037
- const inputDataForRecording = _formatInputData(client, args);
1038
- currentTrace.recordInput(inputDataForRecording);
1039
- try {
1040
- // 2. Make the API call
1041
- response = yield boundOriginalMethod(...args);
1042
- }
1043
- catch (error) {
1044
- currentTrace.recordError(error);
1045
- throw error;
1046
- }
1047
- // 3. Extract output data *after* await
1048
- const outputData = _formatOutputData(client, response);
1049
- // --- Calculate and Embed Costs (Reorganized) ---
1050
- // 4. Extract modelName and usage *immediately* before the check
1051
- // Get modelName directly from the *original arguments* `args`
1052
- // Get usage from the `outputData` extracted above
1053
- const modelName = (_a = args[0]) === null || _a === void 0 ? void 0 : _a.model; // More direct extraction from original args
1054
- const traceManager = currentTrace.traceManager;
1055
- // 5. Perform the check and cost calculation
1056
- if (modelName && (response === null || response === void 0 ? void 0 : response.usage) && traceManager &&
1057
- (response.usage.prompt_tokens !== undefined || response.usage.input_tokens !== undefined) &&
1058
- (response.usage.completion_tokens !== undefined || response.usage.output_tokens !== undefined)) {
1059
- // Use response.usage directly for calculations too
1060
- const currentUsage = response.usage;
1061
- const promptTokens = (_c = (_b = currentUsage.prompt_tokens) !== null && _b !== void 0 ? _b : currentUsage.input_tokens) !== null && _c !== void 0 ? _c : 0;
1062
- const completionTokens = (_e = (_d = currentUsage.completion_tokens) !== null && _d !== void 0 ? _d : currentUsage.output_tokens) !== null && _e !== void 0 ? _e : 0;
1063
- try {
1064
- const costResponse = yield traceManager.calculateTokenCosts(modelName, promptTokens, completionTokens);
1065
- if (costResponse) {
1066
- // Merge costs into the usage object *within outputData*
1067
- // Need to ensure outputData exists and has usage if we modify it
1068
- if (outputData) { // Check if outputData was created
1069
- outputData.usage = Object.assign(Object.assign({}, currentUsage), { prompt_tokens_cost_usd: costResponse.prompt_tokens_cost_usd, completion_tokens_cost_usd: costResponse.completion_tokens_cost_usd, total_cost_usd: costResponse.total_cost_usd });
1070
- }
1071
- logger_instance_js_1.default.info(`[wrap] Calculated cost for ${modelName}: $${costResponse.total_cost_usd.toFixed(6)}`);
1072
- }
1073
- else {
1074
- logger_instance_js_1.default.warn(`[wrap] Could not fetch token costs for model ${modelName}. Costs will be missing for this span.`);
1075
- }
1076
- }
1077
- catch (costError) {
1078
- logger_instance_js_1.default.error(`[wrap] Error fetching token costs for ${modelName}:`, costError);
1079
- }
1080
- }
1081
- // --- End Cost Calculation ---
1082
- // 6. Record the potentially enriched outputData
1083
- currentTrace.recordOutput(outputData);
1084
- } // End span loop
1085
- // The response is returned outside the loop, after the span context has ended
1086
- return response;
1087
- });
1088
- // Generic patching
1089
- if (methodOwner && methodName && typeof methodOwner[methodName] === 'function') {
1090
- methodOwner[methodName] = tracedMethod;
1091
- logger_instance_js_1.default.info(`Successfully wrapped method '${methodName}' on owner '${((_a = methodOwner.constructor) === null || _a === void 0 ? void 0 : _a.name) || '(unknown)'}' for client: ${(_b = client === null || client === void 0 ? void 0 : client.constructor) === null || _b === void 0 ? void 0 : _b.name}`);
1092
- }
1093
- else {
1094
- logger_instance_js_1.default.error("Failed to apply wrapper: Could not find method owner or method name.", { clientType: (_c = client === null || client === void 0 ? void 0 : client.constructor) === null || _c === void 0 ? void 0 : _c.name, methodName });
1095
- return client; // Return unwrapped if patching fails
1096
- }
1097
- return client;
1098
- }
1099
- //# sourceMappingURL=tracer.js.map