@zukhruf/evals 0.0.0-stage → 0.8.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (356) hide show
  1. package/LICENSE +21 -0
  2. package/NOTICE +82 -0
  3. package/README.md +540 -2
  4. package/dist/aggregation/counts.d.ts +11 -0
  5. package/dist/aggregation/counts.d.ts.map +1 -0
  6. package/dist/aggregation/counts.js +14 -0
  7. package/dist/aggregation/counts.js.map +1 -0
  8. package/dist/aggregation/f-measure-form.d.ts +11 -0
  9. package/dist/aggregation/f-measure-form.d.ts.map +1 -0
  10. package/dist/aggregation/f-measure-form.js +25 -0
  11. package/dist/aggregation/f-measure-form.js.map +1 -0
  12. package/dist/aggregation/f-measure-ratio.d.ts +12 -0
  13. package/dist/aggregation/f-measure-ratio.d.ts.map +1 -0
  14. package/dist/aggregation/f-measure-ratio.js +21 -0
  15. package/dist/aggregation/f-measure-ratio.js.map +1 -0
  16. package/dist/aggregation/ratio-of-sums.d.ts +9 -0
  17. package/dist/aggregation/ratio-of-sums.d.ts.map +1 -0
  18. package/dist/aggregation/ratio-of-sums.js +28 -0
  19. package/dist/aggregation/ratio-of-sums.js.map +1 -0
  20. package/dist/aggregation/ratio.d.ts +8 -0
  21. package/dist/aggregation/ratio.d.ts.map +1 -0
  22. package/dist/aggregation/ratio.js +20 -0
  23. package/dist/aggregation/ratio.js.map +1 -0
  24. package/dist/classification/all-class-matches.d.ts +7 -0
  25. package/dist/classification/all-class-matches.d.ts.map +1 -0
  26. package/dist/classification/all-class-matches.js +14 -0
  27. package/dist/classification/all-class-matches.js.map +1 -0
  28. package/dist/classification/class-matches.d.ts +10 -0
  29. package/dist/classification/class-matches.d.ts.map +1 -0
  30. package/dist/classification/class-matches.js +21 -0
  31. package/dist/classification/class-matches.js.map +1 -0
  32. package/dist/classification/index.d.ts +11 -0
  33. package/dist/classification/index.d.ts.map +1 -0
  34. package/dist/classification/index.js +10 -0
  35. package/dist/classification/index.js.map +1 -0
  36. package/dist/classification/label-case.d.ts +11 -0
  37. package/dist/classification/label-case.d.ts.map +1 -0
  38. package/dist/classification/label-case.js +12 -0
  39. package/dist/classification/label-case.js.map +1 -0
  40. package/dist/classification/label-comparer.d.ts +8 -0
  41. package/dist/classification/label-comparer.d.ts.map +1 -0
  42. package/dist/classification/label-comparer.js +44 -0
  43. package/dist/classification/label-comparer.js.map +1 -0
  44. package/dist/classification/label-comparison.d.ts +12 -0
  45. package/dist/classification/label-comparison.d.ts.map +1 -0
  46. package/dist/classification/label-comparison.js +23 -0
  47. package/dist/classification/label-comparison.js.map +1 -0
  48. package/dist/classification/label-config-case.d.ts +6 -0
  49. package/dist/classification/label-config-case.d.ts.map +1 -0
  50. package/dist/classification/label-config-case.js +13 -0
  51. package/dist/classification/label-config-case.js.map +1 -0
  52. package/dist/classification/label-vars-case.d.ts +6 -0
  53. package/dist/classification/label-vars-case.d.ts.map +1 -0
  54. package/dist/classification/label-vars-case.js +17 -0
  55. package/dist/classification/label-vars-case.js.map +1 -0
  56. package/dist/classification/label.d.ts +4 -0
  57. package/dist/classification/label.d.ts.map +1 -0
  58. package/dist/classification/label.js +16 -0
  59. package/dist/classification/label.js.map +1 -0
  60. package/dist/classification/labels-match.d.ts +7 -0
  61. package/dist/classification/labels-match.d.ts.map +1 -0
  62. package/dist/classification/labels-match.js +13 -0
  63. package/dist/classification/labels-match.js.map +1 -0
  64. package/dist/classification/per-class.d.ts +12 -0
  65. package/dist/classification/per-class.d.ts.map +1 -0
  66. package/dist/classification/per-class.js +17 -0
  67. package/dist/classification/per-class.js.map +1 -0
  68. package/dist/classification/read-labels.d.ts +3 -0
  69. package/dist/classification/read-labels.d.ts.map +1 -0
  70. package/dist/classification/read-labels.js +11 -0
  71. package/dist/classification/read-labels.js.map +1 -0
  72. package/dist/index.d.ts +25 -0
  73. package/dist/index.d.ts.map +1 -0
  74. package/dist/index.js +20 -0
  75. package/dist/index.js.map +1 -0
  76. package/dist/promptfoo/assertion-module.d.ts +23 -0
  77. package/dist/promptfoo/assertion-module.d.ts.map +1 -0
  78. package/dist/promptfoo/assertion-module.js +45 -0
  79. package/dist/promptfoo/assertion-module.js.map +1 -0
  80. package/dist/promptfoo/index.d.ts +4 -0
  81. package/dist/promptfoo/index.d.ts.map +1 -0
  82. package/dist/promptfoo/index.js +4 -0
  83. package/dist/promptfoo/index.js.map +1 -0
  84. package/dist/promptfoo/promptfoo-assertion.d.ts +18 -0
  85. package/dist/promptfoo/promptfoo-assertion.d.ts.map +1 -0
  86. package/dist/promptfoo/promptfoo-assertion.js +31 -0
  87. package/dist/promptfoo/promptfoo-assertion.js.map +1 -0
  88. package/dist/promptfoo/scorer-assertion.d.ts +18 -0
  89. package/dist/promptfoo/scorer-assertion.d.ts.map +1 -0
  90. package/dist/promptfoo/scorer-assertion.js +28 -0
  91. package/dist/promptfoo/scorer-assertion.js.map +1 -0
  92. package/dist/scorers/at-least.d.ts +9 -0
  93. package/dist/scorers/at-least.d.ts.map +1 -0
  94. package/dist/scorers/at-least.js +18 -0
  95. package/dist/scorers/at-least.js.map +1 -0
  96. package/dist/scorers/f-beta.d.ts +13 -0
  97. package/dist/scorers/f-beta.d.ts.map +1 -0
  98. package/dist/scorers/f-beta.js +22 -0
  99. package/dist/scorers/f-beta.js.map +1 -0
  100. package/dist/scorers/f1.d.ts +9 -0
  101. package/dist/scorers/f1.d.ts.map +1 -0
  102. package/dist/scorers/f1.js +8 -0
  103. package/dist/scorers/f1.js.map +1 -0
  104. package/dist/scorers/gate.d.ts +7 -0
  105. package/dist/scorers/gate.d.ts.map +1 -0
  106. package/dist/scorers/gate.js +10 -0
  107. package/dist/scorers/gate.js.map +1 -0
  108. package/dist/scorers/matches.d.ts +5 -0
  109. package/dist/scorers/matches.d.ts.map +1 -0
  110. package/dist/scorers/matches.js +2 -0
  111. package/dist/scorers/matches.js.map +1 -0
  112. package/dist/scorers/mean-of.d.ts +8 -0
  113. package/dist/scorers/mean-of.d.ts.map +1 -0
  114. package/dist/scorers/mean-of.js +15 -0
  115. package/dist/scorers/mean-of.js.map +1 -0
  116. package/dist/scorers/penalized.d.ts +9 -0
  117. package/dist/scorers/penalized.d.ts.map +1 -0
  118. package/dist/scorers/penalized.js +29 -0
  119. package/dist/scorers/penalized.js.map +1 -0
  120. package/dist/scorers/penalty.d.ts +7 -0
  121. package/dist/scorers/penalty.d.ts.map +1 -0
  122. package/dist/scorers/penalty.js +11 -0
  123. package/dist/scorers/penalty.js.map +1 -0
  124. package/dist/scorers/precision.d.ts +9 -0
  125. package/dist/scorers/precision.d.ts.map +1 -0
  126. package/dist/scorers/precision.js +13 -0
  127. package/dist/scorers/precision.js.map +1 -0
  128. package/dist/scorers/recall.d.ts +9 -0
  129. package/dist/scorers/recall.d.ts.map +1 -0
  130. package/dist/scorers/recall.js +13 -0
  131. package/dist/scorers/recall.js.map +1 -0
  132. package/dist/scorers/sample-averaged.d.ts +10 -0
  133. package/dist/scorers/sample-averaged.d.ts.map +1 -0
  134. package/dist/scorers/sample-averaged.js +17 -0
  135. package/dist/scorers/sample-averaged.js.map +1 -0
  136. package/dist/scorers/scorer.d.ts +5 -0
  137. package/dist/scorers/scorer.d.ts.map +1 -0
  138. package/dist/scorers/scorer.js +2 -0
  139. package/dist/scorers/scorer.js.map +1 -0
  140. package/dist/scorers/shortfall-penalty.d.ts +11 -0
  141. package/dist/scorers/shortfall-penalty.d.ts.map +1 -0
  142. package/dist/scorers/shortfall-penalty.js +25 -0
  143. package/dist/scorers/shortfall-penalty.js.map +1 -0
  144. package/dist/scoring/bootstrap.d.ts +16 -0
  145. package/dist/scoring/bootstrap.d.ts.map +1 -0
  146. package/dist/scoring/bootstrap.js +29 -0
  147. package/dist/scoring/bootstrap.js.map +1 -0
  148. package/dist/scoring/comparer.d.ts +5 -0
  149. package/dist/scoring/comparer.d.ts.map +1 -0
  150. package/dist/scoring/comparer.js +2 -0
  151. package/dist/scoring/comparer.js.map +1 -0
  152. package/dist/scoring/grouped.d.ts +9 -0
  153. package/dist/scoring/grouped.d.ts.map +1 -0
  154. package/dist/scoring/grouped.js +20 -0
  155. package/dist/scoring/grouped.js.map +1 -0
  156. package/dist/scoring/scorecard-scorer.d.ts +11 -0
  157. package/dist/scoring/scorecard-scorer.d.ts.map +1 -0
  158. package/dist/scoring/scorecard-scorer.js +2 -0
  159. package/dist/scoring/scorecard-scorer.js.map +1 -0
  160. package/dist/scoring/scorecard.d.ts +15 -0
  161. package/dist/scoring/scorecard.d.ts.map +1 -0
  162. package/dist/scoring/scorecard.js +2 -0
  163. package/dist/scoring/scorecard.js.map +1 -0
  164. package/dist/scoring/scorer-set.d.ts +13 -0
  165. package/dist/scoring/scorer-set.d.ts.map +1 -0
  166. package/dist/scoring/scorer-set.js +55 -0
  167. package/dist/scoring/scorer-set.js.map +1 -0
  168. package/dist/scoring/totals.d.ts +9 -0
  169. package/dist/scoring/totals.d.ts.map +1 -0
  170. package/dist/scoring/totals.js +23 -0
  171. package/dist/scoring/totals.js.map +1 -0
  172. package/dist/structured/comparison/alignment.d.ts +9 -0
  173. package/dist/structured/comparison/alignment.d.ts.map +1 -0
  174. package/dist/structured/comparison/alignment.js +2 -0
  175. package/dist/structured/comparison/alignment.js.map +1 -0
  176. package/dist/structured/comparison/alignments/by-best-match.d.ts +9 -0
  177. package/dist/structured/comparison/alignments/by-best-match.d.ts.map +1 -0
  178. package/dist/structured/comparison/alignments/by-best-match.js +25 -0
  179. package/dist/structured/comparison/alignments/by-best-match.js.map +1 -0
  180. package/dist/structured/comparison/alignments/by-position.d.ts +6 -0
  181. package/dist/structured/comparison/alignments/by-position.d.ts.map +1 -0
  182. package/dist/structured/comparison/alignments/by-position.js +9 -0
  183. package/dist/structured/comparison/alignments/by-position.js.map +1 -0
  184. package/dist/structured/comparison/bag-of-words.d.ts +10 -0
  185. package/dist/structured/comparison/bag-of-words.d.ts.map +1 -0
  186. package/dist/structured/comparison/bag-of-words.js +48 -0
  187. package/dist/structured/comparison/bag-of-words.js.map +1 -0
  188. package/dist/structured/comparison/comparator.d.ts +5 -0
  189. package/dist/structured/comparison/comparator.d.ts.map +1 -0
  190. package/dist/structured/comparison/comparator.js +2 -0
  191. package/dist/structured/comparison/comparator.js.map +1 -0
  192. package/dist/structured/comparison/comparators/json-equality.d.ts +7 -0
  193. package/dist/structured/comparison/comparators/json-equality.d.ts.map +1 -0
  194. package/dist/structured/comparison/comparators/json-equality.js +18 -0
  195. package/dist/structured/comparison/comparators/json-equality.js.map +1 -0
  196. package/dist/structured/comparison/comparators/normalized-text.d.ts +7 -0
  197. package/dist/structured/comparison/comparators/normalized-text.d.ts.map +1 -0
  198. package/dist/structured/comparison/comparators/normalized-text.js +13 -0
  199. package/dist/structured/comparison/comparators/normalized-text.js.map +1 -0
  200. package/dist/structured/comparison/comparators/normalized.d.ts +2 -0
  201. package/dist/structured/comparison/comparators/normalized.d.ts.map +1 -0
  202. package/dist/structured/comparison/comparators/normalized.js +3 -0
  203. package/dist/structured/comparison/comparators/normalized.js.map +1 -0
  204. package/dist/structured/comparison/comparators/number-within.d.ts +9 -0
  205. package/dist/structured/comparison/comparators/number-within.d.ts.map +1 -0
  206. package/dist/structured/comparison/comparators/number-within.js +18 -0
  207. package/dist/structured/comparison/comparators/number-within.js.map +1 -0
  208. package/dist/structured/comparison/comparators/python-equality.d.ts +7 -0
  209. package/dist/structured/comparison/comparators/python-equality.d.ts.map +1 -0
  210. package/dist/structured/comparison/comparators/python-equality.js +15 -0
  211. package/dist/structured/comparison/comparators/python-equality.js.map +1 -0
  212. package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts +12 -0
  213. package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts.map +1 -0
  214. package/dist/structured/comparison/comparators/token-overlap-at-least.js +23 -0
  215. package/dist/structured/comparison/comparators/token-overlap-at-least.js.map +1 -0
  216. package/dist/structured/comparison/comparators/typed-comparator.d.ts +8 -0
  217. package/dist/structured/comparison/comparators/typed-comparator.d.ts.map +1 -0
  218. package/dist/structured/comparison/comparators/typed-comparator.js +10 -0
  219. package/dist/structured/comparison/comparators/typed-comparator.js.map +1 -0
  220. package/dist/structured/comparison/comparison-spec.d.ts +17 -0
  221. package/dist/structured/comparison/comparison-spec.d.ts.map +1 -0
  222. package/dist/structured/comparison/comparison-spec.js +36 -0
  223. package/dist/structured/comparison/comparison-spec.js.map +1 -0
  224. package/dist/structured/comparison/comparison.d.ts +25 -0
  225. package/dist/structured/comparison/comparison.d.ts.map +1 -0
  226. package/dist/structured/comparison/comparison.js +94 -0
  227. package/dist/structured/comparison/comparison.js.map +1 -0
  228. package/dist/structured/comparison/field.d.ts +10 -0
  229. package/dist/structured/comparison/field.d.ts.map +1 -0
  230. package/dist/structured/comparison/field.js +29 -0
  231. package/dist/structured/comparison/field.js.map +1 -0
  232. package/dist/structured/comparison/item-pairs.d.ts +8 -0
  233. package/dist/structured/comparison/item-pairs.d.ts.map +1 -0
  234. package/dist/structured/comparison/item-pairs.js +2 -0
  235. package/dist/structured/comparison/item-pairs.js.map +1 -0
  236. package/dist/structured/comparison/json-path-query.d.ts +7 -0
  237. package/dist/structured/comparison/json-path-query.d.ts.map +1 -0
  238. package/dist/structured/comparison/json-path-query.js +133 -0
  239. package/dist/structured/comparison/json-path-query.js.map +1 -0
  240. package/dist/structured/comparison/json-value.d.ts +11 -0
  241. package/dist/structured/comparison/json-value.d.ts.map +1 -0
  242. package/dist/structured/comparison/json-value.js +56 -0
  243. package/dist/structured/comparison/json-value.js.map +1 -0
  244. package/dist/structured/comparison/leaf-pair.d.ts +24 -0
  245. package/dist/structured/comparison/leaf-pair.d.ts.map +1 -0
  246. package/dist/structured/comparison/leaf-pair.js +2 -0
  247. package/dist/structured/comparison/leaf-pair.js.map +1 -0
  248. package/dist/structured/comparison/match.d.ts +5 -0
  249. package/dist/structured/comparison/match.d.ts.map +1 -0
  250. package/dist/structured/comparison/match.js +2 -0
  251. package/dist/structured/comparison/match.js.map +1 -0
  252. package/dist/structured/comparison/name-selector.d.ts +6 -0
  253. package/dist/structured/comparison/name-selector.d.ts.map +1 -0
  254. package/dist/structured/comparison/name-selector.js +41 -0
  255. package/dist/structured/comparison/name-selector.js.map +1 -0
  256. package/dist/structured/comparison/path.d.ts +10 -0
  257. package/dist/structured/comparison/path.d.ts.map +1 -0
  258. package/dist/structured/comparison/path.js +27 -0
  259. package/dist/structured/comparison/path.js.map +1 -0
  260. package/dist/structured/comparison/tree-walk.d.ts +10 -0
  261. package/dist/structured/comparison/tree-walk.d.ts.map +1 -0
  262. package/dist/structured/comparison/tree-walk.js +140 -0
  263. package/dist/structured/comparison/tree-walk.js.map +1 -0
  264. package/dist/structured/cpython/python-text.d.ts +4 -0
  265. package/dist/structured/cpython/python-text.d.ts.map +1 -0
  266. package/dist/structured/cpython/python-text.js +62 -0
  267. package/dist/structured/cpython/python-text.js.map +1 -0
  268. package/dist/structured/index.d.ts +37 -0
  269. package/dist/structured/index.d.ts.map +1 -0
  270. package/dist/structured/index.js +27 -0
  271. package/dist/structured/index.js.map +1 -0
  272. package/dist/structured/schema/draft-07-schema.d.ts +3 -0
  273. package/dist/structured/schema/draft-07-schema.d.ts.map +1 -0
  274. package/dist/structured/schema/draft-07-schema.js +243 -0
  275. package/dist/structured/schema/draft-07-schema.js.map +1 -0
  276. package/dist/structured/schema/draft-2020-12-schema.d.ts +4 -0
  277. package/dist/structured/schema/draft-2020-12-schema.d.ts.map +1 -0
  278. package/dist/structured/schema/draft-2020-12-schema.js +280 -0
  279. package/dist/structured/schema/draft-2020-12-schema.js.map +1 -0
  280. package/dist/structured/schema/is-record.d.ts +2 -0
  281. package/dist/structured/schema/is-record.d.ts.map +1 -0
  282. package/dist/structured/schema/is-record.js +3 -0
  283. package/dist/structured/schema/is-record.js.map +1 -0
  284. package/dist/structured/schema/json-schema.d.ts +20 -0
  285. package/dist/structured/schema/json-schema.d.ts.map +1 -0
  286. package/dist/structured/schema/json-schema.js +244 -0
  287. package/dist/structured/schema/json-schema.js.map +1 -0
  288. package/dist/structured/schema/schema-type-walk.d.ts +11 -0
  289. package/dist/structured/schema/schema-type-walk.d.ts.map +1 -0
  290. package/dist/structured/schema/schema-type-walk.js +196 -0
  291. package/dist/structured/schema/schema-type-walk.js.map +1 -0
  292. package/dist/structured/scorers/all-matched.d.ts +10 -0
  293. package/dist/structured/scorers/all-matched.d.ts.map +1 -0
  294. package/dist/structured/scorers/all-matched.js +36 -0
  295. package/dist/structured/scorers/all-matched.js.map +1 -0
  296. package/dist/structured/scorers/exact-match.d.ts +7 -0
  297. package/dist/structured/scorers/exact-match.d.ts.map +1 -0
  298. package/dist/structured/scorers/exact-match.js +13 -0
  299. package/dist/structured/scorers/exact-match.js.map +1 -0
  300. package/dist/structured/scorers/item-matches.d.ts +7 -0
  301. package/dist/structured/scorers/item-matches.d.ts.map +1 -0
  302. package/dist/structured/scorers/item-matches.js +17 -0
  303. package/dist/structured/scorers/item-matches.js.map +1 -0
  304. package/dist/structured/scorers/leaf-matches.d.ts +14 -0
  305. package/dist/structured/scorers/leaf-matches.d.ts.map +1 -0
  306. package/dist/structured/scorers/leaf-matches.js +28 -0
  307. package/dist/structured/scorers/leaf-matches.js.map +1 -0
  308. package/dist/structured/scorers/parse-success.d.ts +7 -0
  309. package/dist/structured/scorers/parse-success.d.ts.map +1 -0
  310. package/dist/structured/scorers/parse-success.js +11 -0
  311. package/dist/structured/scorers/parse-success.js.map +1 -0
  312. package/dist/structured/scorers/path-matches.d.ts +5 -0
  313. package/dist/structured/scorers/path-matches.d.ts.map +1 -0
  314. package/dist/structured/scorers/path-matches.js +9 -0
  315. package/dist/structured/scorers/path-matches.js.map +1 -0
  316. package/dist/structured/scorers/required-key-recall.d.ts +8 -0
  317. package/dist/structured/scorers/required-key-recall.d.ts.map +1 -0
  318. package/dist/structured/scorers/required-key-recall.js +50 -0
  319. package/dist/structured/scorers/required-key-recall.js.map +1 -0
  320. package/dist/structured/scorers/schema-compliance.d.ts +8 -0
  321. package/dist/structured/scorers/schema-compliance.d.ts.map +1 -0
  322. package/dist/structured/scorers/schema-compliance.js +29 -0
  323. package/dist/structured/scorers/schema-compliance.js.map +1 -0
  324. package/dist/structured/scorers/token-f1.d.ts +16 -0
  325. package/dist/structured/scorers/token-f1.d.ts.map +1 -0
  326. package/dist/structured/scorers/token-f1.js +47 -0
  327. package/dist/structured/scorers/token-f1.js.map +1 -0
  328. package/dist/structured/scorers/type-precision.d.ts +10 -0
  329. package/dist/structured/scorers/type-precision.d.ts.map +1 -0
  330. package/dist/structured/scorers/type-precision.js +30 -0
  331. package/dist/structured/scorers/type-precision.js.map +1 -0
  332. package/dist/structured/scorers/valid-schema.d.ts +7 -0
  333. package/dist/structured/scorers/valid-schema.d.ts.map +1 -0
  334. package/dist/structured/scorers/valid-schema.js +11 -0
  335. package/dist/structured/scorers/valid-schema.js.map +1 -0
  336. package/dist/structured/scorers/value-matches.d.ts +6 -0
  337. package/dist/structured/scorers/value-matches.d.ts.map +1 -0
  338. package/dist/structured/scorers/value-matches.js +9 -0
  339. package/dist/structured/scorers/value-matches.js.map +1 -0
  340. package/dist/structured/scoring/case.d.ts +14 -0
  341. package/dist/structured/scoring/case.d.ts.map +1 -0
  342. package/dist/structured/scoring/case.js +14 -0
  343. package/dist/structured/scoring/case.js.map +1 -0
  344. package/dist/structured/scoring/config-case.d.ts +11 -0
  345. package/dist/structured/scoring/config-case.d.ts.map +1 -0
  346. package/dist/structured/scoring/config-case.js +22 -0
  347. package/dist/structured/scoring/config-case.js.map +1 -0
  348. package/dist/structured/scoring/per-field.d.ts +6 -0
  349. package/dist/structured/scoring/per-field.d.ts.map +1 -0
  350. package/dist/structured/scoring/per-field.js +14 -0
  351. package/dist/structured/scoring/per-field.js.map +1 -0
  352. package/dist/structured/scoring/text-vars-case.d.ts +12 -0
  353. package/dist/structured/scoring/text-vars-case.d.ts.map +1 -0
  354. package/dist/structured/scoring/text-vars-case.js +31 -0
  355. package/dist/structured/scoring/text-vars-case.js.map +1 -0
  356. package/package.json +84 -4
@@ -0,0 +1,17 @@
1
+ import { Grouped } from "../scoring/grouped.js";
2
+ // A scorer for each class of a comparison: in a ScorerSet, it gives the ratio
3
+ // of its scorer on the comparison of each class, under the label
4
+ // `${class} ${label}`. Without labels, the classes are the labels of both
5
+ // sides of the answer; with labels, only those classes.
6
+ export class PerClass extends Grouped {
7
+ #labels;
8
+ constructor(scorer, { labels } = {}) {
9
+ super(scorer);
10
+ this.#labels = labels;
11
+ }
12
+ *groups(comparison) {
13
+ for (const label of this.#labels ?? comparison.labels())
14
+ yield [`${label}`, comparison.only(label)];
15
+ }
16
+ }
17
+ //# sourceMappingURL=per-class.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"per-class.js","sourceRoot":"","sources":["../../src/classification/per-class.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAIhD,8EAA8E;AAC9E,iEAAiE;AACjE,0EAA0E;AAC1E,wDAAwD;AACxD,MAAM,OAAO,QAAS,SAAQ,OAAwB;IAC3C,OAAO,CAA+B;IAE/C,YACE,MAA+B,EAC/B,EAAE,MAAM,KAA6C,EAAE;QAEvD,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAES,CAAC,MAAM,CACf,UAA2B;QAE3B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC,MAAM,EAAE;YACrD,MAAM,CAAC,GAAG,KAAK,EAAE,EAAE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/C,CAAC;CACF"}
@@ -0,0 +1,3 @@
1
+ import { type Label } from './label.ts';
2
+ export declare function readLabels(value: unknown, name: string): Label | Label[];
3
+ //# sourceMappingURL=read-labels.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"read-labels.d.ts","sourceRoot":"","sources":["../../src/classification/read-labels.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,KAAK,EAAW,MAAM,YAAY,CAAC;AAIjD,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,EAAE,CAMxE"}
@@ -0,0 +1,11 @@
1
+ import { isLabel } from "./label.js";
2
+ // The expected labels of a plain object of a runner: a label or a list of
3
+ // labels.
4
+ export function readLabels(value, name) {
5
+ if (isLabel(value))
6
+ return value;
7
+ if (Array.isArray(value) && value.every(isLabel))
8
+ return value;
9
+ throw new Error(`${name} is a label (a number or a string) or a list of labels, not ${JSON.stringify(value)}`);
10
+ }
11
+ //# sourceMappingURL=read-labels.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"read-labels.js","sourceRoot":"","sources":["../../src/classification/read-labels.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,OAAO,EAAE,MAAM,YAAY,CAAC;AAEjD,0EAA0E;AAC1E,UAAU;AACV,MAAM,UAAU,UAAU,CAAC,KAAc,EAAE,IAAY;IACrD,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACjC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/D,MAAM,IAAI,KAAK,CACb,GAAG,IAAI,+DAA+D,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAC9F,CAAC;AACJ,CAAC"}
@@ -0,0 +1,25 @@
1
+ export { Ratio } from './aggregation/ratio.ts';
2
+ export { RatioOfSums } from './aggregation/ratio-of-sums.ts';
3
+ export { Counts } from './aggregation/counts.ts';
4
+ export { CountsForm, type FMeasureForm, PrecisionRecallForm, } from './aggregation/f-measure-form.ts';
5
+ export type { Check, Scorecard } from './scoring/scorecard.ts';
6
+ export type { CaseReader, ScorecardScorer, Weighted, } from './scoring/scorecard-scorer.ts';
7
+ export { Totals } from './scoring/totals.ts';
8
+ export { Bootstrap, type WeightedScorecard } from './scoring/bootstrap.ts';
9
+ export type { Scorer } from './scorers/scorer.ts';
10
+ export type { Matches } from './scorers/matches.ts';
11
+ export { Precision } from './scorers/precision.ts';
12
+ export { Recall } from './scorers/recall.ts';
13
+ export { FBeta } from './scorers/f-beta.ts';
14
+ export { F1 } from './scorers/f1.ts';
15
+ export { Penalty } from './scorers/penalty.ts';
16
+ export { Gate } from './scorers/gate.ts';
17
+ export { Penalized } from './scorers/penalized.ts';
18
+ export { AtLeast } from './scorers/at-least.ts';
19
+ export { ShortfallPenalty } from './scorers/shortfall-penalty.ts';
20
+ export { SampleAveraged } from './scorers/sample-averaged.ts';
21
+ export { MeanOf } from './scorers/mean-of.ts';
22
+ export type { Comparer } from './scoring/comparer.ts';
23
+ export { Grouped } from './scoring/grouped.ts';
24
+ export { ScorerSet } from './scoring/scorer-set.ts';
25
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EACL,UAAU,EACV,KAAK,YAAY,EACjB,mBAAmB,GACpB,MAAM,iCAAiC,CAAC;AACzC,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAC/D,YAAY,EACV,UAAU,EACV,eAAe,EACf,QAAQ,GACT,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,KAAK,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,YAAY,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,YAAY,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAC5C,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,YAAY,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
package/dist/index.js ADDED
@@ -0,0 +1,20 @@
1
+ export { Ratio } from "./aggregation/ratio.js";
2
+ export { RatioOfSums } from "./aggregation/ratio-of-sums.js";
3
+ export { Counts } from "./aggregation/counts.js";
4
+ export { CountsForm, PrecisionRecallForm, } from "./aggregation/f-measure-form.js";
5
+ export { Totals } from "./scoring/totals.js";
6
+ export { Bootstrap } from "./scoring/bootstrap.js";
7
+ export { Precision } from "./scorers/precision.js";
8
+ export { Recall } from "./scorers/recall.js";
9
+ export { FBeta } from "./scorers/f-beta.js";
10
+ export { F1 } from "./scorers/f1.js";
11
+ export { Penalty } from "./scorers/penalty.js";
12
+ export { Gate } from "./scorers/gate.js";
13
+ export { Penalized } from "./scorers/penalized.js";
14
+ export { AtLeast } from "./scorers/at-least.js";
15
+ export { ShortfallPenalty } from "./scorers/shortfall-penalty.js";
16
+ export { SampleAveraged } from "./scorers/sample-averaged.js";
17
+ export { MeanOf } from "./scorers/mean-of.js";
18
+ export { Grouped } from "./scoring/grouped.js";
19
+ export { ScorerSet } from "./scoring/scorer-set.js";
20
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EACL,UAAU,EAEV,mBAAmB,GACpB,MAAM,iCAAiC,CAAC;AAOzC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAG3E,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAC5C,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAE9C,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
@@ -0,0 +1,23 @@
1
+ export type VarsContext = {
2
+ readonly vars: Readonly<Record<string, unknown>>;
3
+ };
4
+ export type ModuleAssertion = {
5
+ readonly metric?: string | undefined;
6
+ readonly weight: number;
7
+ readonly threshold: number;
8
+ readonly value: (output: string, context: VarsContext) => number;
9
+ };
10
+ export type FileAssertion = {
11
+ readonly type: 'javascript';
12
+ readonly metric: string;
13
+ readonly weight: number;
14
+ readonly threshold: number;
15
+ readonly value: string;
16
+ };
17
+ export declare class AssertionModule<Name extends string> {
18
+ #private;
19
+ readonly functions: Readonly<Record<string, ModuleAssertion['value']>>;
20
+ constructor(url: string, assertions: Readonly<Record<Name, ModuleAssertion>>);
21
+ assert(...names: Name[]): FileAssertion[];
22
+ }
23
+ //# sourceMappingURL=assertion-module.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"assertion-module.d.ts","sourceRoot":"","sources":["../../src/promptfoo/assertion-module.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CAClD,CAAC;AAKF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,KAAK,MAAM,CAAC;CAClE,CAAC;AAKF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAgBF,qBAAa,eAAe,CAAC,IAAI,SAAS,MAAM;;IAG9C,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAKrE,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;IAcrD,MAAM,CAAC,GAAG,KAAK,EAAE,IAAI,EAAE,GAAG,aAAa,EAAE;CAY1C"}
@@ -0,0 +1,45 @@
1
+ import { fileURLToPath } from 'node:url';
2
+ // The module that promptfoo loads the function of each scorer assertion from
3
+ // (ADR 0013). promptfoo stores the config of an eval as JSON, and --resume
4
+ // and --retry-errors build the run from that stored config, so a function in
5
+ // the config is lost and its assertion scores 0. A config holds the file
6
+ // references that assert gives, and promptfoo loads each function from the
7
+ // default export of the module: importModule reads file://<module>:<name> on
8
+ // the default export when the module has one (promptfoo 0.125.1,
9
+ // dist/src/esm-B4NX_h4K.js:205-224). So the module gives functions as its
10
+ // default export, and url is the module's own import.meta.url.
11
+ //
12
+ // The reference holds the absolute path of the module. promptfoo 0.125.1
13
+ // splits a file reference at its first colon
14
+ // (dist/src/evaluator-BxCmvGtv.js:5683-5686), so on Windows the drive of the
15
+ // path ends the path: a defect of promptfoo.
16
+ export class AssertionModule {
17
+ // promptfoo reads a function by the name in its reference, so the names are
18
+ // keys, and assert holds the type of each name.
19
+ functions;
20
+ #path;
21
+ #assertions;
22
+ constructor(url, assertions) {
23
+ this.#path = fileURLToPath(url);
24
+ this.#assertions = assertions;
25
+ this.functions = Object.fromEntries(Object.entries(assertions).map(([name, assertion]) => [
26
+ name,
27
+ assertion.value,
28
+ ]));
29
+ }
30
+ // One assertion for each name. The name is the metric, unless its scorer
31
+ // assertion gives one.
32
+ assert(...names) {
33
+ return names.map((name) => {
34
+ const { metric = name, weight, threshold } = this.#assertions[name];
35
+ return {
36
+ type: 'javascript',
37
+ metric,
38
+ weight,
39
+ threshold,
40
+ value: `file://${this.#path}:${name}`,
41
+ };
42
+ });
43
+ }
44
+ }
45
+ //# sourceMappingURL=assertion-module.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"assertion-module.js","sourceRoot":"","sources":["../../src/promptfoo/assertion-module.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AA6BzC,6EAA6E;AAC7E,2EAA2E;AAC3E,6EAA6E;AAC7E,yEAAyE;AACzE,2EAA2E;AAC3E,6EAA6E;AAC7E,iEAAiE;AACjE,0EAA0E;AAC1E,+DAA+D;AAC/D,EAAE;AACF,yEAAyE;AACzE,6CAA6C;AAC7C,6EAA6E;AAC7E,6CAA6C;AAC7C,MAAM,OAAO,eAAe;IAC1B,4EAA4E;IAC5E,gDAAgD;IACvC,SAAS,CAAqD;IAC9D,KAAK,CAAS;IACd,WAAW,CAA0C;IAE9D,YACE,GAAW,EACX,UAAmD;QAEnD,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,WAAW,CACjC,MAAM,CAAC,OAAO,CAAkB,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC;YACrE,IAAI;YACJ,SAAS,CAAC,KAAK;SAChB,CAAC,CACH,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,uBAAuB;IACvB,MAAM,CAAC,GAAG,KAAa;QACrB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACxB,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACpE,OAAO;gBACL,IAAI,EAAE,YAAY;gBAClB,MAAM;gBACN,MAAM;gBACN,SAAS;gBACT,KAAK,EAAE,UAAU,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE;aACtC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
@@ -0,0 +1,4 @@
1
+ export { AssertionModule, type FileAssertion, type ModuleAssertion, type VarsContext, } from './assertion-module.ts';
2
+ export { type AssertionContext, type GradingResult, PromptfooAssertion, } from './promptfoo-assertion.ts';
3
+ export { ScorerAssertion } from './scorer-assertion.ts';
4
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/promptfoo/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,WAAW,GACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAClB,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC"}
@@ -0,0 +1,4 @@
1
+ export { AssertionModule, } from "./assertion-module.js";
2
+ export { PromptfooAssertion, } from "./promptfoo-assertion.js";
3
+ export { ScorerAssertion } from "./scorer-assertion.js";
4
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/promptfoo/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,GAIhB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAGL,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC"}
@@ -0,0 +1,18 @@
1
+ import type { CaseReader, ScorecardScorer, Weighted } from '../scoring/scorecard-scorer.ts';
2
+ export type GradingResult = {
3
+ readonly pass: boolean;
4
+ readonly score: number;
5
+ readonly reason: string;
6
+ readonly namedScores?: Readonly<Record<string, number>>;
7
+ readonly namedScoreWeights?: Readonly<Record<string, number>>;
8
+ readonly componentResults?: readonly GradingResult[];
9
+ };
10
+ export type AssertionContext = {
11
+ readonly config?: Readonly<Record<string, unknown>>;
12
+ };
13
+ export declare class PromptfooAssertion<Case extends Weighted> {
14
+ #private;
15
+ constructor(scorer: ScorecardScorer<Case>, cases: CaseReader<Case>);
16
+ readonly grade: (output: unknown, context: AssertionContext) => GradingResult;
17
+ }
18
+ //# sourceMappingURL=promptfoo-assertion.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"promptfoo-assertion.d.ts","sourceRoot":"","sources":["../../src/promptfoo/promptfoo-assertion.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EACV,eAAe,EACf,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAKxC,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9D,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;CACtD,CAAC;AAIF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACrD,CAAC;AAOF,qBAAa,kBAAkB,CAAC,IAAI,SAAS,QAAQ;;gBAIvC,MAAM,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC;IAKlE,QAAQ,CAAC,KAAK,GACZ,QAAQ,OAAO,EACf,SAAS,gBAAgB,KACxB,aAAa,CAsBd;CACH"}
@@ -0,0 +1,31 @@
1
+ // Adapts a scorer to a promptfoo javascript assertion (ADR 0004). Each ratio
2
+ // of the scorecard becomes a named score with its whole as the weight:
3
+ // promptfoo adds value × weight over the rows and divides by the weights
4
+ // (namedMetrics.ts, accumulateNamedMetric), which is the ratio of sums.
5
+ // promptfoo calls a function, so a module exports `grade`.
6
+ export class PromptfooAssertion {
7
+ #scorer;
8
+ #cases;
9
+ constructor(scorer, cases) {
10
+ this.#scorer = scorer;
11
+ this.#cases = cases;
12
+ }
13
+ grade = (output, context) => {
14
+ const testCase = this.#cases.from(context.config ?? {});
15
+ const scorecard = this.#scorer.score(output, testCase);
16
+ const ratios = [...scorecard.ratios].map(([name, ratio]) => [name, ratio.times(testCase.weight)]);
17
+ return {
18
+ pass: scorecard.passed,
19
+ score: scorecard.score,
20
+ reason: scorecard.reason,
21
+ namedScores: Object.fromEntries(ratios.map(([name, ratio]) => [name, ratio.value])),
22
+ namedScoreWeights: Object.fromEntries(ratios.map(([name, ratio]) => [name, ratio.whole])),
23
+ componentResults: scorecard.checks.map((check) => ({
24
+ pass: check.passed,
25
+ score: check.score,
26
+ reason: check.reason,
27
+ })),
28
+ };
29
+ };
30
+ }
31
+ //# sourceMappingURL=promptfoo-assertion.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"promptfoo-assertion.js","sourceRoot":"","sources":["../../src/promptfoo/promptfoo-assertion.ts"],"names":[],"mappings":"AAwBA,6EAA6E;AAC7E,uEAAuE;AACvE,yEAAyE;AACzE,wEAAwE;AACxE,2DAA2D;AAC3D,MAAM,OAAO,kBAAkB;IACpB,OAAO,CAAwB;IAC/B,MAAM,CAAmB;IAElC,YAAY,MAA6B,EAAE,KAAuB;QAChE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAEQ,KAAK,GAAG,CACf,MAAe,EACf,OAAyB,EACV,EAAE;QACjB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,CACtC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAU,CACjE,CAAC;QACF,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,WAAW,EAAE,MAAM,CAAC,WAAW,CAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CACnD;YACD,iBAAiB,EAAE,MAAM,CAAC,WAAW,CACnC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CACnD;YACD,gBAAgB,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACjD,IAAI,EAAE,KAAK,CAAC,MAAM;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,MAAM,EAAE,KAAK,CAAC,MAAM;aACrB,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC,CAAC;CACH"}
@@ -0,0 +1,18 @@
1
+ import type { Scorer } from '../scorers/scorer.ts';
2
+ import type { Comparer } from '../scoring/comparer.ts';
3
+ import type { CaseReader, Weighted } from '../scoring/scorecard-scorer.ts';
4
+ import type { VarsContext } from './assertion-module.ts';
5
+ export declare class ScorerAssertion<Case extends Weighted, Input> {
6
+ readonly metric: string | undefined;
7
+ readonly weight: number;
8
+ readonly threshold: number;
9
+ readonly value: (output: string, context: VarsContext) => number;
10
+ constructor(scorer: Scorer<Input>, { weight, threshold, metric, comparer, cases, }: {
11
+ readonly weight: number;
12
+ readonly threshold: number;
13
+ readonly metric?: string;
14
+ readonly comparer: Comparer<Case, Input>;
15
+ readonly cases: CaseReader<Case>;
16
+ });
17
+ }
18
+ //# sourceMappingURL=scorer-assertion.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scorer-assertion.d.ts","sourceRoot":"","sources":["../../src/promptfoo/scorer-assertion.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,gCAAgC,CAAC;AAC3E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAiBzD,qBAAa,eAAe,CAAC,IAAI,SAAS,QAAQ,EAAE,KAAK;IACvD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,KAAK,MAAM,CAAC;gBAG/D,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EACrB,EACE,MAAM,EACN,SAAS,EACT,MAAM,EACN,QAAQ,EACR,KAAK,GACN,EAAE;QACD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACzC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;KAClC;CAQJ"}
@@ -0,0 +1,28 @@
1
+ // One scorer with the rule of its promptfoo assertion (ADR 0007, ADR 0013):
2
+ // how much it counts in the score of the test (weight: the score is the sum
3
+ // of value × weight over the sum of the weights), when it passes (threshold:
4
+ // a number passes when it reaches it), and the metric it reports under when
5
+ // that is not its name in its assertion module (metric: promptfoo averages
6
+ // the values of one metric over the tests, which is a metric in the sense of
7
+ // ADR 0009). It is not a promptfoo assertion: promptfoo stores a config as
8
+ // JSON, so a config gets a file reference from its AssertionModule, and
9
+ // promptfoo loads value from the module. value gets the text of the harness,
10
+ // and gives the value of the ratio of the scorer: promptfoo takes the mean of
11
+ // the values of one metric, so a SampleAveraged scorer gives its samples
12
+ // average. The case comes from the vars of the test, as its case reader reads
13
+ // them: an assertion that defaultTest shares gets only the vars of each test.
14
+ // The comparer compares the answer with the case and gives the input of the
15
+ // scorer.
16
+ export class ScorerAssertion {
17
+ metric;
18
+ weight;
19
+ threshold;
20
+ value;
21
+ constructor(scorer, { weight, threshold, metric, comparer, cases, }) {
22
+ this.metric = metric;
23
+ this.weight = weight;
24
+ this.threshold = threshold;
25
+ this.value = (output, { vars }) => scorer.score(comparer.compare(output, cases.from(vars))).value;
26
+ }
27
+ }
28
+ //# sourceMappingURL=scorer-assertion.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scorer-assertion.js","sourceRoot":"","sources":["../../src/promptfoo/scorer-assertion.ts"],"names":[],"mappings":"AAKA,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,4EAA4E;AAC5E,2EAA2E;AAC3E,6EAA6E;AAC7E,2EAA2E;AAC3E,wEAAwE;AACxE,6EAA6E;AAC7E,8EAA8E;AAC9E,yEAAyE;AACzE,8EAA8E;AAC9E,8EAA8E;AAC9E,4EAA4E;AAC5E,UAAU;AACV,MAAM,OAAO,eAAe;IACjB,MAAM,CAAqB;IAC3B,MAAM,CAAS;IACf,SAAS,CAAS;IAClB,KAAK,CAAmD;IAEjE,YACE,MAAqB,EACrB,EACE,MAAM,EACN,SAAS,EACT,MAAM,EACN,QAAQ,EACR,KAAK,GAON;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAChC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACnE,CAAC;CACF"}
@@ -0,0 +1,9 @@
1
+ import { Gate } from './gate.ts';
2
+ import type { Scorer } from './scorer.ts';
3
+ export declare class AtLeast<Input> extends Gate<Input> {
4
+ #private;
5
+ constructor(scorer: Scorer<Input>, threshold: number);
6
+ holds(input: Input): boolean;
7
+ explain(input: Input): string;
8
+ }
9
+ //# sourceMappingURL=at-least.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"at-least.d.ts","sourceRoot":"","sources":["../../src/scorers/at-least.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,qBAAa,OAAO,CAAC,KAAK,CAAE,SAAQ,IAAI,CAAC,KAAK,CAAC;;gBAIjC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM;IAMpD,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO;IAI5B,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;CAG9B"}
@@ -0,0 +1,18 @@
1
+ import { Gate } from "./gate.js";
2
+ // A gate: it holds when the value of the scorer reaches the threshold.
3
+ export class AtLeast extends Gate {
4
+ #scorer;
5
+ #threshold;
6
+ constructor(scorer, threshold) {
7
+ super();
8
+ this.#scorer = scorer;
9
+ this.#threshold = threshold;
10
+ }
11
+ holds(input) {
12
+ return this.#scorer.score(input).value >= this.#threshold;
13
+ }
14
+ explain(input) {
15
+ return `the value ${this.#scorer.score(input).value} is below the threshold ${this.#threshold}`;
16
+ }
17
+ }
18
+ //# sourceMappingURL=at-least.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"at-least.js","sourceRoot":"","sources":["../../src/scorers/at-least.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,uEAAuE;AACvE,MAAM,OAAO,OAAe,SAAQ,IAAW;IACpC,OAAO,CAAgB;IACvB,UAAU,CAAS;IAE5B,YAAY,MAAqB,EAAE,SAAiB;QAClD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC;IAC5D,CAAC;IAED,OAAO,CAAC,KAAY;QAClB,OAAO,aAAa,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,2BAA2B,IAAI,CAAC,UAAU,EAAE,CAAC;IAClG,CAAC;CACF"}
@@ -0,0 +1,13 @@
1
+ import { type FMeasureForm } from '../aggregation/f-measure-form.ts';
2
+ import type { Ratio } from '../aggregation/ratio.ts';
3
+ import type { Matches } from './matches.ts';
4
+ import type { Scorer } from './scorer.ts';
5
+ export declare class FBeta<Input> implements Scorer<Input> {
6
+ #private;
7
+ constructor(matches: Matches<Input>, { beta, form, }: {
8
+ readonly beta: number;
9
+ readonly form?: FMeasureForm;
10
+ });
11
+ score(input: Input): Ratio;
12
+ }
13
+ //# sourceMappingURL=f-beta.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"f-beta.d.ts","sourceRoot":"","sources":["../../src/scorers/f-beta.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,kCAAkC,CAAC;AAE1C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAK1C,qBAAa,KAAK,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAM9C,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,EACvB,EACE,IAAI,EACJ,IAAuB,GACxB,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAA;KAAE;IAO5D,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAM3B"}
@@ -0,0 +1,22 @@
1
+ import { CountsForm, } from "../aggregation/f-measure-form.js";
2
+ import { FMeasureRatio } from "../aggregation/f-measure-ratio.js";
3
+ // The F-beta of the hits, as scikit-learn's fbeta_score: β weighs recall β
4
+ // times as much as precision. The form is the arithmetic of the value, and
5
+ // CountsForm is the form of scikit-learn.
6
+ export class FBeta {
7
+ #matches;
8
+ #beta;
9
+ #form;
10
+ constructor(matches, { beta, form = new CountsForm(), }) {
11
+ this.#matches = matches;
12
+ this.#beta = beta;
13
+ this.#form = form;
14
+ }
15
+ score(input) {
16
+ return new FMeasureRatio(this.#matches.counts(input), {
17
+ beta: this.#beta,
18
+ form: this.#form,
19
+ });
20
+ }
21
+ }
22
+ //# sourceMappingURL=f-beta.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"f-beta.js","sourceRoot":"","sources":["../../src/scorers/f-beta.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,GAEX,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAKlE,2EAA2E;AAC3E,2EAA2E;AAC3E,0CAA0C;AAC1C,MAAM,OAAO,KAAK;IACP,QAAQ,CAAiB;IACzB,KAAK,CAAS;IACd,KAAK,CAAe;IAE7B,YACE,OAAuB,EACvB,EACE,IAAI,EACJ,IAAI,GAAG,IAAI,UAAU,EAAE,GACiC;QAE1D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACpD,IAAI,EAAE,IAAI,CAAC,KAAK;YAChB,IAAI,EAAE,IAAI,CAAC,KAAK;SACjB,CAAC,CAAC;IACL,CAAC;CACF"}
@@ -0,0 +1,9 @@
1
+ import type { FMeasureForm } from '../aggregation/f-measure-form.ts';
2
+ import { FBeta } from './f-beta.ts';
3
+ import type { Matches } from './matches.ts';
4
+ export declare class F1<Input> extends FBeta<Input> {
5
+ constructor(matches: Matches<Input>, options?: {
6
+ readonly form?: FMeasureForm;
7
+ });
8
+ }
9
+ //# sourceMappingURL=f1.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"f1.d.ts","sourceRoot":"","sources":["../../src/scorers/f1.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAG5C,qBAAa,EAAE,CAAC,KAAK,CAAE,SAAQ,KAAK,CAAC,KAAK,CAAC;gBAEvC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,EACvB,OAAO,GAAE;QAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAA;KAAO;CAIjD"}
@@ -0,0 +1,8 @@
1
+ import { FBeta } from "./f-beta.js";
2
+ // The F1 of the hits: the F-beta with β = 1.
3
+ export class F1 extends FBeta {
4
+ constructor(matches, options = {}) {
5
+ super(matches, { ...options, beta: 1 });
6
+ }
7
+ }
8
+ //# sourceMappingURL=f1.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"f1.js","sourceRoot":"","sources":["../../src/scorers/f1.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAGpC,6CAA6C;AAC7C,MAAM,OAAO,EAAU,SAAQ,KAAY;IACzC,YACE,OAAuB,EACvB,UAA4C,EAAE;QAE9C,KAAK,CAAC,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC;CACF"}
@@ -0,0 +1,7 @@
1
+ import { Penalty } from './penalty.ts';
2
+ export declare abstract class Gate<Input> extends Penalty<Input> {
3
+ abstract holds(input: Input): boolean;
4
+ abstract explain(input: Input): string;
5
+ factor(input: Input): number;
6
+ }
7
+ //# sourceMappingURL=gate.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"gate.d.ts","sourceRoot":"","sources":["../../src/scorers/gate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAKvC,8BAAsB,IAAI,CAAC,KAAK,CAAE,SAAQ,OAAO,CAAC,KAAK,CAAC;IACtD,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO;IAGrC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;IAEtC,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;CAG7B"}
@@ -0,0 +1,10 @@
1
+ import { Penalty } from "./penalty.js";
2
+ // The penalty whose factor is 1 when the input holds what the gate checks,
3
+ // else 0. A subclass says what it checks and why an input fails it (Template
4
+ // Method), so a ScorerSet makes a check of each gate.
5
+ export class Gate extends Penalty {
6
+ factor(input) {
7
+ return this.holds(input) ? 1 : 0;
8
+ }
9
+ }
10
+ //# sourceMappingURL=gate.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"gate.js","sourceRoot":"","sources":["../../src/scorers/gate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,2EAA2E;AAC3E,6EAA6E;AAC7E,sDAAsD;AACtD,MAAM,OAAgB,IAAY,SAAQ,OAAc;IAMtD,MAAM,CAAC,KAAY;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;CACF"}
@@ -0,0 +1,5 @@
1
+ import type { Counts } from '../aggregation/counts.ts';
2
+ export interface Matches<Input> {
3
+ counts(input: Input): Counts;
4
+ }
5
+ //# sourceMappingURL=matches.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"matches.d.ts","sourceRoot":"","sources":["../../src/scorers/matches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIvD,MAAM,WAAW,OAAO,CAAC,KAAK;IAC5B,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;CAC9B"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=matches.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"matches.js","sourceRoot":"","sources":["../../src/scorers/matches.ts"],"names":[],"mappings":""}
@@ -0,0 +1,8 @@
1
+ import { Ratio } from '../aggregation/ratio.ts';
2
+ import type { Scorer } from './scorer.ts';
3
+ export declare class MeanOf<Input> implements Scorer<Input> {
4
+ #private;
5
+ constructor(scorer: Scorer<Input>, ...scorers: readonly Scorer<Input>[]);
6
+ score(input: Input): Ratio;
7
+ }
8
+ //# sourceMappingURL=mean-of.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"mean-of.d.ts","sourceRoot":"","sources":["../../src/scorers/mean-of.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAK1C,qBAAa,MAAM,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAGrC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,MAAM,CAAC,KAAK,CAAC,EAAE;IAIvE,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAQ3B"}
@@ -0,0 +1,15 @@
1
+ import { NeumaierSum } from '@zukhruf/python';
2
+ import { Ratio } from "../aggregation/ratio.js";
3
+ // The mean of the values of some scorers for one record, over 1, with
4
+ // CPython's sum, such as a category of a benchmark. It takes one scorer or
5
+ // more, because the mean of no value has no value.
6
+ export class MeanOf {
7
+ #scorers;
8
+ constructor(scorer, ...scorers) {
9
+ this.#scorers = [scorer, ...scorers];
10
+ }
11
+ score(input) {
12
+ return new Ratio(NeumaierSum.mean(this.#scorers.map((scorer) => scorer.score(input).value)), 1);
13
+ }
14
+ }
15
+ //# sourceMappingURL=mean-of.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"mean-of.js","sourceRoot":"","sources":["../../src/scorers/mean-of.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGhD,sEAAsE;AACtE,2EAA2E;AAC3E,mDAAmD;AACnD,MAAM,OAAO,MAAM;IACR,QAAQ,CAA2B;IAE5C,YAAY,MAAqB,EAAE,GAAG,OAAiC;QACrE,IAAI,CAAC,QAAQ,GAAG,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,KAAK,CACd,WAAW,CAAC,IAAI,CACd,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CACzD,EACD,CAAC,CACF,CAAC;IACJ,CAAC;CACF"}
@@ -0,0 +1,9 @@
1
+ import { Ratio } from '../aggregation/ratio.ts';
2
+ import type { Penalty } from './penalty.ts';
3
+ import type { Scorer } from './scorer.ts';
4
+ export declare class Penalized<Input> implements Scorer<Input> {
5
+ #private;
6
+ constructor(scorer: Scorer<Input>, ...penalties: readonly Penalty<Input>[]);
7
+ score(input: Input): Ratio;
8
+ }
9
+ //# sourceMappingURL=penalized.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"penalized.d.ts","sourceRoot":"","sources":["../../src/scorers/penalized.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAS1C,qBAAa,SAAS,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAIxC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,SAAS,EAAE,SAAS,OAAO,CAAC,KAAK,CAAC,EAAE;IAK1E,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAgB3B"}
@@ -0,0 +1,29 @@
1
+ import { Ratio } from "../aggregation/ratio.js";
2
+ // A scorer whose part is multiplied by the factor of each penalty, in the
3
+ // order of the penalties, for each record; the whole stays (Decorator). So a
4
+ // record that fails a gate keeps its expected values in the denominator of
5
+ // recall. A penalty on the means would give mean(a) × mean(b), so the
6
+ // penalty is per record (ADR 0004). When every factor is 1, the ratio of the
7
+ // scorer stays as it is, so an F-measure keeps the value of its form (ADR
8
+ // 0012).
9
+ export class Penalized {
10
+ #scorer;
11
+ #penalties;
12
+ constructor(scorer, ...penalties) {
13
+ this.#scorer = scorer;
14
+ this.#penalties = penalties;
15
+ }
16
+ score(input) {
17
+ const ratio = this.#scorer.score(input);
18
+ const factors = this.#penalties.map((penalty) => {
19
+ const factor = penalty.factor(input);
20
+ if (!(factor >= 0 && factor <= 1))
21
+ throw new RangeError(`A penalty gave the factor ${factor}, which is not in [0, 1].`);
22
+ return factor;
23
+ });
24
+ if (factors.every((factor) => factor === 1))
25
+ return ratio;
26
+ return new Ratio(factors.reduce((part, factor) => part * factor, ratio.part), ratio.whole);
27
+ }
28
+ }
29
+ //# sourceMappingURL=penalized.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"penalized.js","sourceRoot":"","sources":["../../src/scorers/penalized.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAIhD,0EAA0E;AAC1E,6EAA6E;AAC7E,2EAA2E;AAC3E,sEAAsE;AACtE,6EAA6E;AAC7E,0EAA0E;AAC1E,SAAS;AACT,MAAM,OAAO,SAAS;IACX,OAAO,CAAgB;IACvB,UAAU,CAA4B;IAE/C,YAAY,MAAqB,EAAE,GAAG,SAAoC;QACxE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,CAAC;gBAC/B,MAAM,IAAI,UAAU,CAClB,6BAA6B,MAAM,2BAA2B,CAC/D,CAAC;YACJ,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAC1D,OAAO,IAAI,KAAK,CACd,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,GAAG,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,EAC3D,KAAK,CAAC,KAAK,CACZ,CAAC;IACJ,CAAC;CACF"}
@@ -0,0 +1,7 @@
1
+ import { Ratio } from '../aggregation/ratio.ts';
2
+ import type { Scorer } from './scorer.ts';
3
+ export declare abstract class Penalty<Input> implements Scorer<Input> {
4
+ abstract factor(input: Input): number;
5
+ score(input: Input): Ratio;
6
+ }
7
+ //# sourceMappingURL=penalty.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"penalty.d.ts","sourceRoot":"","sources":["../../src/scorers/penalty.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAM1C,8BAAsB,OAAO,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;IAC3D,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;IAErC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAG3B"}
@@ -0,0 +1,11 @@
1
+ import { Ratio } from "../aggregation/ratio.js";
2
+ // A factor in [0, 1] for one input, as BLEU's brevity penalty is: 1 is no
3
+ // penalty. Penalized multiplies the part of a scorer by it, for each record.
4
+ // A subclass gives the factor (Template Method). As a scorer, it gives the
5
+ // factor over 1.
6
+ export class Penalty {
7
+ score(input) {
8
+ return new Ratio(this.factor(input), 1);
9
+ }
10
+ }
11
+ //# sourceMappingURL=penalty.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"penalty.js","sourceRoot":"","sources":["../../src/scorers/penalty.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGhD,0EAA0E;AAC1E,6EAA6E;AAC7E,2EAA2E;AAC3E,iBAAiB;AACjB,MAAM,OAAgB,OAAO;IAG3B,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC;CACF"}