@zukhruf/evals 0.0.0-stage → 0.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (292) hide show
  1. package/LICENSE +21 -0
  2. package/NOTICE +82 -0
  3. package/README.md +528 -2
  4. package/dist/aggregation/counts.d.ts +11 -0
  5. package/dist/aggregation/counts.d.ts.map +1 -0
  6. package/dist/aggregation/counts.js +14 -0
  7. package/dist/aggregation/counts.js.map +1 -0
  8. package/dist/aggregation/f1-ratio.d.ts +8 -0
  9. package/dist/aggregation/f1-ratio.d.ts.map +1 -0
  10. package/dist/aggregation/f1-ratio.js +24 -0
  11. package/dist/aggregation/f1-ratio.js.map +1 -0
  12. package/dist/aggregation/ratio-of-sums.d.ts +8 -0
  13. package/dist/aggregation/ratio-of-sums.d.ts.map +1 -0
  14. package/dist/aggregation/ratio-of-sums.js +21 -0
  15. package/dist/aggregation/ratio-of-sums.js.map +1 -0
  16. package/dist/aggregation/ratio.d.ts +8 -0
  17. package/dist/aggregation/ratio.d.ts.map +1 -0
  18. package/dist/aggregation/ratio.js +20 -0
  19. package/dist/aggregation/ratio.js.map +1 -0
  20. package/dist/index.d.ts +21 -0
  21. package/dist/index.d.ts.map +1 -0
  22. package/dist/index.js +17 -0
  23. package/dist/index.js.map +1 -0
  24. package/dist/promptfoo/assertion-module.d.ts +23 -0
  25. package/dist/promptfoo/assertion-module.d.ts.map +1 -0
  26. package/dist/promptfoo/assertion-module.js +45 -0
  27. package/dist/promptfoo/assertion-module.js.map +1 -0
  28. package/dist/promptfoo/index.d.ts +3 -0
  29. package/dist/promptfoo/index.d.ts.map +1 -0
  30. package/dist/promptfoo/index.js +3 -0
  31. package/dist/promptfoo/index.js.map +1 -0
  32. package/dist/promptfoo/promptfoo-assertion.d.ts +18 -0
  33. package/dist/promptfoo/promptfoo-assertion.d.ts.map +1 -0
  34. package/dist/promptfoo/promptfoo-assertion.js +31 -0
  35. package/dist/promptfoo/promptfoo-assertion.js.map +1 -0
  36. package/dist/scorers/at-least.d.ts +9 -0
  37. package/dist/scorers/at-least.d.ts.map +1 -0
  38. package/dist/scorers/at-least.js +18 -0
  39. package/dist/scorers/at-least.js.map +1 -0
  40. package/dist/scorers/f1.d.ts +9 -0
  41. package/dist/scorers/f1.d.ts.map +1 -0
  42. package/dist/scorers/f1.js +12 -0
  43. package/dist/scorers/f1.js.map +1 -0
  44. package/dist/scorers/gate.d.ts +7 -0
  45. package/dist/scorers/gate.d.ts.map +1 -0
  46. package/dist/scorers/gate.js +10 -0
  47. package/dist/scorers/gate.js.map +1 -0
  48. package/dist/scorers/matches.d.ts +5 -0
  49. package/dist/scorers/matches.d.ts.map +1 -0
  50. package/dist/scorers/matches.js +2 -0
  51. package/dist/scorers/matches.js.map +1 -0
  52. package/dist/scorers/mean-of.d.ts +8 -0
  53. package/dist/scorers/mean-of.d.ts.map +1 -0
  54. package/dist/scorers/mean-of.js +15 -0
  55. package/dist/scorers/mean-of.js.map +1 -0
  56. package/dist/scorers/penalized.d.ts +9 -0
  57. package/dist/scorers/penalized.d.ts.map +1 -0
  58. package/dist/scorers/penalized.js +28 -0
  59. package/dist/scorers/penalized.js.map +1 -0
  60. package/dist/scorers/penalty.d.ts +7 -0
  61. package/dist/scorers/penalty.d.ts.map +1 -0
  62. package/dist/scorers/penalty.js +11 -0
  63. package/dist/scorers/penalty.js.map +1 -0
  64. package/dist/scorers/precision.d.ts +9 -0
  65. package/dist/scorers/precision.d.ts.map +1 -0
  66. package/dist/scorers/precision.js +13 -0
  67. package/dist/scorers/precision.js.map +1 -0
  68. package/dist/scorers/recall.d.ts +9 -0
  69. package/dist/scorers/recall.d.ts.map +1 -0
  70. package/dist/scorers/recall.js +13 -0
  71. package/dist/scorers/recall.js.map +1 -0
  72. package/dist/scorers/sample-averaged.d.ts +10 -0
  73. package/dist/scorers/sample-averaged.d.ts.map +1 -0
  74. package/dist/scorers/sample-averaged.js +17 -0
  75. package/dist/scorers/sample-averaged.js.map +1 -0
  76. package/dist/scorers/scorer.d.ts +5 -0
  77. package/dist/scorers/scorer.d.ts.map +1 -0
  78. package/dist/scorers/scorer.js +2 -0
  79. package/dist/scorers/scorer.js.map +1 -0
  80. package/dist/scorers/shortfall-penalty.d.ts +11 -0
  81. package/dist/scorers/shortfall-penalty.d.ts.map +1 -0
  82. package/dist/scorers/shortfall-penalty.js +25 -0
  83. package/dist/scorers/shortfall-penalty.js.map +1 -0
  84. package/dist/scoring/bootstrap.d.ts +16 -0
  85. package/dist/scoring/bootstrap.d.ts.map +1 -0
  86. package/dist/scoring/bootstrap.js +31 -0
  87. package/dist/scoring/bootstrap.js.map +1 -0
  88. package/dist/scoring/scorecard-scorer.d.ts +11 -0
  89. package/dist/scoring/scorecard-scorer.d.ts.map +1 -0
  90. package/dist/scoring/scorecard-scorer.js +2 -0
  91. package/dist/scoring/scorecard-scorer.js.map +1 -0
  92. package/dist/scoring/scorecard.d.ts +15 -0
  93. package/dist/scoring/scorecard.d.ts.map +1 -0
  94. package/dist/scoring/scorecard.js +2 -0
  95. package/dist/scoring/scorecard.js.map +1 -0
  96. package/dist/scoring/totals.d.ts +9 -0
  97. package/dist/scoring/totals.d.ts.map +1 -0
  98. package/dist/scoring/totals.js +23 -0
  99. package/dist/scoring/totals.js.map +1 -0
  100. package/dist/structured/comparison/alignment.d.ts +9 -0
  101. package/dist/structured/comparison/alignment.d.ts.map +1 -0
  102. package/dist/structured/comparison/alignment.js +2 -0
  103. package/dist/structured/comparison/alignment.js.map +1 -0
  104. package/dist/structured/comparison/alignments/by-best-match.d.ts +9 -0
  105. package/dist/structured/comparison/alignments/by-best-match.d.ts.map +1 -0
  106. package/dist/structured/comparison/alignments/by-best-match.js +25 -0
  107. package/dist/structured/comparison/alignments/by-best-match.js.map +1 -0
  108. package/dist/structured/comparison/alignments/by-position.d.ts +6 -0
  109. package/dist/structured/comparison/alignments/by-position.d.ts.map +1 -0
  110. package/dist/structured/comparison/alignments/by-position.js +9 -0
  111. package/dist/structured/comparison/alignments/by-position.js.map +1 -0
  112. package/dist/structured/comparison/bag-of-words.d.ts +10 -0
  113. package/dist/structured/comparison/bag-of-words.d.ts.map +1 -0
  114. package/dist/structured/comparison/bag-of-words.js +48 -0
  115. package/dist/structured/comparison/bag-of-words.js.map +1 -0
  116. package/dist/structured/comparison/comparator.d.ts +5 -0
  117. package/dist/structured/comparison/comparator.d.ts.map +1 -0
  118. package/dist/structured/comparison/comparator.js +2 -0
  119. package/dist/structured/comparison/comparator.js.map +1 -0
  120. package/dist/structured/comparison/comparators/json-equality.d.ts +7 -0
  121. package/dist/structured/comparison/comparators/json-equality.d.ts.map +1 -0
  122. package/dist/structured/comparison/comparators/json-equality.js +18 -0
  123. package/dist/structured/comparison/comparators/json-equality.js.map +1 -0
  124. package/dist/structured/comparison/comparators/normalized-text.d.ts +7 -0
  125. package/dist/structured/comparison/comparators/normalized-text.d.ts.map +1 -0
  126. package/dist/structured/comparison/comparators/normalized-text.js +13 -0
  127. package/dist/structured/comparison/comparators/normalized-text.js.map +1 -0
  128. package/dist/structured/comparison/comparators/normalized.d.ts +2 -0
  129. package/dist/structured/comparison/comparators/normalized.d.ts.map +1 -0
  130. package/dist/structured/comparison/comparators/normalized.js +3 -0
  131. package/dist/structured/comparison/comparators/normalized.js.map +1 -0
  132. package/dist/structured/comparison/comparators/number-within.d.ts +9 -0
  133. package/dist/structured/comparison/comparators/number-within.d.ts.map +1 -0
  134. package/dist/structured/comparison/comparators/number-within.js +18 -0
  135. package/dist/structured/comparison/comparators/number-within.js.map +1 -0
  136. package/dist/structured/comparison/comparators/python-equality.d.ts +7 -0
  137. package/dist/structured/comparison/comparators/python-equality.d.ts.map +1 -0
  138. package/dist/structured/comparison/comparators/python-equality.js +15 -0
  139. package/dist/structured/comparison/comparators/python-equality.js.map +1 -0
  140. package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts +9 -0
  141. package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts.map +1 -0
  142. package/dist/structured/comparison/comparators/token-overlap-at-least.js +20 -0
  143. package/dist/structured/comparison/comparators/token-overlap-at-least.js.map +1 -0
  144. package/dist/structured/comparison/comparators/typed-comparator.d.ts +8 -0
  145. package/dist/structured/comparison/comparators/typed-comparator.d.ts.map +1 -0
  146. package/dist/structured/comparison/comparators/typed-comparator.js +10 -0
  147. package/dist/structured/comparison/comparators/typed-comparator.js.map +1 -0
  148. package/dist/structured/comparison/comparison-spec.d.ts +17 -0
  149. package/dist/structured/comparison/comparison-spec.d.ts.map +1 -0
  150. package/dist/structured/comparison/comparison-spec.js +36 -0
  151. package/dist/structured/comparison/comparison-spec.js.map +1 -0
  152. package/dist/structured/comparison/comparison.d.ts +25 -0
  153. package/dist/structured/comparison/comparison.d.ts.map +1 -0
  154. package/dist/structured/comparison/comparison.js +94 -0
  155. package/dist/structured/comparison/comparison.js.map +1 -0
  156. package/dist/structured/comparison/field.d.ts +10 -0
  157. package/dist/structured/comparison/field.d.ts.map +1 -0
  158. package/dist/structured/comparison/field.js +29 -0
  159. package/dist/structured/comparison/field.js.map +1 -0
  160. package/dist/structured/comparison/item-pairs.d.ts +8 -0
  161. package/dist/structured/comparison/item-pairs.d.ts.map +1 -0
  162. package/dist/structured/comparison/item-pairs.js +2 -0
  163. package/dist/structured/comparison/item-pairs.js.map +1 -0
  164. package/dist/structured/comparison/json-path-query.d.ts +7 -0
  165. package/dist/structured/comparison/json-path-query.d.ts.map +1 -0
  166. package/dist/structured/comparison/json-path-query.js +133 -0
  167. package/dist/structured/comparison/json-path-query.js.map +1 -0
  168. package/dist/structured/comparison/json-value.d.ts +10 -0
  169. package/dist/structured/comparison/json-value.d.ts.map +1 -0
  170. package/dist/structured/comparison/json-value.js +47 -0
  171. package/dist/structured/comparison/json-value.js.map +1 -0
  172. package/dist/structured/comparison/leaf-pair.d.ts +24 -0
  173. package/dist/structured/comparison/leaf-pair.d.ts.map +1 -0
  174. package/dist/structured/comparison/leaf-pair.js +2 -0
  175. package/dist/structured/comparison/leaf-pair.js.map +1 -0
  176. package/dist/structured/comparison/match.d.ts +5 -0
  177. package/dist/structured/comparison/match.d.ts.map +1 -0
  178. package/dist/structured/comparison/match.js +2 -0
  179. package/dist/structured/comparison/match.js.map +1 -0
  180. package/dist/structured/comparison/name-selector.d.ts +6 -0
  181. package/dist/structured/comparison/name-selector.d.ts.map +1 -0
  182. package/dist/structured/comparison/name-selector.js +41 -0
  183. package/dist/structured/comparison/name-selector.js.map +1 -0
  184. package/dist/structured/comparison/path.d.ts +10 -0
  185. package/dist/structured/comparison/path.d.ts.map +1 -0
  186. package/dist/structured/comparison/path.js +27 -0
  187. package/dist/structured/comparison/path.js.map +1 -0
  188. package/dist/structured/comparison/tree-walk.d.ts +10 -0
  189. package/dist/structured/comparison/tree-walk.d.ts.map +1 -0
  190. package/dist/structured/comparison/tree-walk.js +136 -0
  191. package/dist/structured/comparison/tree-walk.js.map +1 -0
  192. package/dist/structured/cpython/python-text.d.ts +5 -0
  193. package/dist/structured/cpython/python-text.d.ts.map +1 -0
  194. package/dist/structured/cpython/python-text.js +79 -0
  195. package/dist/structured/cpython/python-text.js.map +1 -0
  196. package/dist/structured/index.d.ts +39 -0
  197. package/dist/structured/index.d.ts.map +1 -0
  198. package/dist/structured/index.js +29 -0
  199. package/dist/structured/index.js.map +1 -0
  200. package/dist/structured/promptfoo/scorer-assertion.d.ts +22 -0
  201. package/dist/structured/promptfoo/scorer-assertion.d.ts.map +1 -0
  202. package/dist/structured/promptfoo/scorer-assertion.js +34 -0
  203. package/dist/structured/promptfoo/scorer-assertion.js.map +1 -0
  204. package/dist/structured/schema/draft-07-schema.d.ts +3 -0
  205. package/dist/structured/schema/draft-07-schema.d.ts.map +1 -0
  206. package/dist/structured/schema/draft-07-schema.js +243 -0
  207. package/dist/structured/schema/draft-07-schema.js.map +1 -0
  208. package/dist/structured/schema/draft-2020-12-schema.d.ts +4 -0
  209. package/dist/structured/schema/draft-2020-12-schema.d.ts.map +1 -0
  210. package/dist/structured/schema/draft-2020-12-schema.js +280 -0
  211. package/dist/structured/schema/draft-2020-12-schema.js.map +1 -0
  212. package/dist/structured/schema/is-record.d.ts +2 -0
  213. package/dist/structured/schema/is-record.d.ts.map +1 -0
  214. package/dist/structured/schema/is-record.js +3 -0
  215. package/dist/structured/schema/is-record.js.map +1 -0
  216. package/dist/structured/schema/json-schema.d.ts +20 -0
  217. package/dist/structured/schema/json-schema.d.ts.map +1 -0
  218. package/dist/structured/schema/json-schema.js +244 -0
  219. package/dist/structured/schema/json-schema.js.map +1 -0
  220. package/dist/structured/schema/schema-type-walk.d.ts +11 -0
  221. package/dist/structured/schema/schema-type-walk.d.ts.map +1 -0
  222. package/dist/structured/schema/schema-type-walk.js +196 -0
  223. package/dist/structured/schema/schema-type-walk.js.map +1 -0
  224. package/dist/structured/scorers/all-matched.d.ts +10 -0
  225. package/dist/structured/scorers/all-matched.d.ts.map +1 -0
  226. package/dist/structured/scorers/all-matched.js +36 -0
  227. package/dist/structured/scorers/all-matched.js.map +1 -0
  228. package/dist/structured/scorers/exact-match.d.ts +7 -0
  229. package/dist/structured/scorers/exact-match.d.ts.map +1 -0
  230. package/dist/structured/scorers/exact-match.js +13 -0
  231. package/dist/structured/scorers/exact-match.js.map +1 -0
  232. package/dist/structured/scorers/item-matches.d.ts +7 -0
  233. package/dist/structured/scorers/item-matches.d.ts.map +1 -0
  234. package/dist/structured/scorers/item-matches.js +17 -0
  235. package/dist/structured/scorers/item-matches.js.map +1 -0
  236. package/dist/structured/scorers/leaf-matches.d.ts +14 -0
  237. package/dist/structured/scorers/leaf-matches.d.ts.map +1 -0
  238. package/dist/structured/scorers/leaf-matches.js +28 -0
  239. package/dist/structured/scorers/leaf-matches.js.map +1 -0
  240. package/dist/structured/scorers/parse-success.d.ts +7 -0
  241. package/dist/structured/scorers/parse-success.d.ts.map +1 -0
  242. package/dist/structured/scorers/parse-success.js +11 -0
  243. package/dist/structured/scorers/parse-success.js.map +1 -0
  244. package/dist/structured/scorers/path-matches.d.ts +5 -0
  245. package/dist/structured/scorers/path-matches.d.ts.map +1 -0
  246. package/dist/structured/scorers/path-matches.js +9 -0
  247. package/dist/structured/scorers/path-matches.js.map +1 -0
  248. package/dist/structured/scorers/required-key-recall.d.ts +8 -0
  249. package/dist/structured/scorers/required-key-recall.d.ts.map +1 -0
  250. package/dist/structured/scorers/required-key-recall.js +50 -0
  251. package/dist/structured/scorers/required-key-recall.js.map +1 -0
  252. package/dist/structured/scorers/schema-compliance.d.ts +8 -0
  253. package/dist/structured/scorers/schema-compliance.d.ts.map +1 -0
  254. package/dist/structured/scorers/schema-compliance.js +29 -0
  255. package/dist/structured/scorers/schema-compliance.js.map +1 -0
  256. package/dist/structured/scorers/token-f1.d.ts +14 -0
  257. package/dist/structured/scorers/token-f1.d.ts.map +1 -0
  258. package/dist/structured/scorers/token-f1.js +44 -0
  259. package/dist/structured/scorers/token-f1.js.map +1 -0
  260. package/dist/structured/scorers/type-precision.d.ts +10 -0
  261. package/dist/structured/scorers/type-precision.d.ts.map +1 -0
  262. package/dist/structured/scorers/type-precision.js +30 -0
  263. package/dist/structured/scorers/type-precision.js.map +1 -0
  264. package/dist/structured/scorers/valid-schema.d.ts +7 -0
  265. package/dist/structured/scorers/valid-schema.d.ts.map +1 -0
  266. package/dist/structured/scorers/valid-schema.js +11 -0
  267. package/dist/structured/scorers/valid-schema.js.map +1 -0
  268. package/dist/structured/scorers/value-matches.d.ts +6 -0
  269. package/dist/structured/scorers/value-matches.d.ts.map +1 -0
  270. package/dist/structured/scorers/value-matches.js +9 -0
  271. package/dist/structured/scorers/value-matches.js.map +1 -0
  272. package/dist/structured/scoring/case.d.ts +14 -0
  273. package/dist/structured/scoring/case.d.ts.map +1 -0
  274. package/dist/structured/scoring/case.js +14 -0
  275. package/dist/structured/scoring/case.js.map +1 -0
  276. package/dist/structured/scoring/config-case.d.ts +11 -0
  277. package/dist/structured/scoring/config-case.d.ts.map +1 -0
  278. package/dist/structured/scoring/config-case.js +22 -0
  279. package/dist/structured/scoring/config-case.js.map +1 -0
  280. package/dist/structured/scoring/per-field.d.ts +9 -0
  281. package/dist/structured/scoring/per-field.d.ts.map +1 -0
  282. package/dist/structured/scoring/per-field.js +23 -0
  283. package/dist/structured/scoring/per-field.js.map +1 -0
  284. package/dist/structured/scoring/scorer-set.d.ts +15 -0
  285. package/dist/structured/scoring/scorer-set.d.ts.map +1 -0
  286. package/dist/structured/scoring/scorer-set.js +60 -0
  287. package/dist/structured/scoring/scorer-set.js.map +1 -0
  288. package/dist/structured/scoring/text-vars-case.d.ts +12 -0
  289. package/dist/structured/scoring/text-vars-case.d.ts.map +1 -0
  290. package/dist/structured/scoring/text-vars-case.js +31 -0
  291. package/dist/structured/scoring/text-vars-case.js.map +1 -0
  292. package/package.json +78 -4
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 ezzabuzaid
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/NOTICE ADDED
@@ -0,0 +1,82 @@
1
+ @zukhruf/structured-eval contains a TypeScript port of the scorer of the
2
+ Structured Output Benchmark (SOB): evaluate.py at commit da785a8 of
3
+ https://github.com/interfazeai/sob. That code has this licence:
4
+
5
+ MIT License
6
+
7
+ Copyright (c) 2026 JigsawStack, Inc.
8
+
9
+ Permission is hereby granted, free of charge, to any person obtaining a copy
10
+ of this software and associated documentation files (the "Software"), to deal
11
+ in the Software without restriction, including without limitation the rights
12
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13
+ copies of the Software, and to permit persons to whom the Software is
14
+ furnished to do so, subject to the following conditions:
15
+
16
+ The above copyright notice and this permission notice shall be included in all
17
+ copies or substantial portions of the Software.
18
+
19
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25
+ SOFTWARE.
26
+
27
+ @zukhruf/structured-eval contains test cases from the JSON-Schema-Test-Suite:
28
+ tests/draft2020-12/optional/ecmascript-regex.json and
29
+ tests/draft7/optional/ecmascript-regex.json at commit
30
+ 245a45abc90190c0bff04d97e6d2b541b4bb9f12 of
31
+ https://github.com/json-schema-org/JSON-Schema-Test-Suite. Those cases have this
32
+ licence:
33
+
34
+ MIT License
35
+
36
+ Copyright (c) 2012 Julian Berman
37
+
38
+ Permission is hereby granted, free of charge, to any person obtaining a copy
39
+ of this software and associated documentation files (the "Software"), to deal
40
+ in the Software without restriction, including without limitation the rights
41
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
42
+ copies of the Software, and to permit persons to whom the Software is
43
+ furnished to do so, subject to the following conditions:
44
+
45
+ The above copyright notice and this permission notice shall be included in
46
+ all copies or substantial portions of the Software.
47
+
48
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
49
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
50
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
51
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
52
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
53
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
54
+ THE SOFTWARE.
55
+
56
+ @zukhruf/structured-eval contains the JSON Schema meta-schemas of draft 7 and
57
+ draft 2020-12 (src/schema/draft-07-schema.ts and
58
+ src/schema/draft-2020-12-schema.ts), copied from jsonschema-specifications
59
+ 2025.9.1 (https://github.com/python-jsonschema/jsonschema-specifications). That
60
+ package has this licence:
61
+
62
+ MIT License
63
+
64
+ Copyright (c) 2022 Julian Berman
65
+
66
+ Permission is hereby granted, free of charge, to any person obtaining a copy
67
+ of this software and associated documentation files (the "Software"), to deal
68
+ in the Software without restriction, including without limitation the rights
69
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
70
+ copies of the Software, and to permit persons to whom the Software is
71
+ furnished to do so, subject to the following conditions:
72
+
73
+ The above copyright notice and this permission notice shall be included in
74
+ all copies or substantial portions of the Software.
75
+
76
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
77
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
78
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
79
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
80
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
81
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
82
+ THE SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,529 @@
1
- # Temporary Holding Version
1
+ # @zukhruf/evals
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ An eval sends inputs to a harness and scores each answer against its label. This package scores the answers. It does not run the harness, load the dataset, store the results or show them. A runner does that work, and the package has an adapter for promptfoo.
4
+
5
+ The answers of evals have different shapes: a structured record, a label, a ranked list, a partition. Each shape has its own rules, so each shape has its own subpath. All subpaths share one core: a scorer gives a ratio for one answer, and a runner adds the ratios of many answers. The words are defined in [CONTEXT.md](CONTEXT.md).
6
+
7
+ ## The parts
8
+
9
+ | Import path | Job |
10
+ | -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
11
+ | `@zukhruf/evals` | The core: scorers, ratios, gates, scorer sets, scorecards, totals and bootstrap intervals. |
12
+ | `@zukhruf/evals/promptfoo` | The adapter that makes a scorer or a scorer set a promptfoo assertion. |
13
+ | `@zukhruf/evals/structured` | Compares a structured value with the value that a person labelled. |
14
+ | `@zukhruf/evals/classification` (next) | Compares the stated labels of an answer with its expected labels, as scikit-learn's `sklearn.metrics` does. Designed in ADR 0003; built in backlog #3889. |
15
+ | `@zukhruf/evals/ranking` (later) | Compares a ranked list with judged items, as trec_eval does. |
16
+ | `@zukhruf/evals/clustering` (later) | Compares a partition with a labelled partition, as the CoNLL reference scorer does. |
17
+
18
+ A subpath imports only the core. It never imports another subpath.
19
+
20
+ The first release (backlog #3888) has one family, so the core holds only the parts that are generic today: `Ratio`, `Counts`, `RatioOfSums`, `Scorecard`, `ScorecardScorer`, `CaseReader`, `Totals` and `Bootstrap`. The scorers, the penalties, `ScorerSet` and `ScorerAssertion` read a structured `Comparison`, so they are in `./structured` until `./classification` (#3889) brings the second input. Then they move into the core with a type parameter, as the API below shows (ADR 0002).
21
+
22
+ ## Score one answer
23
+
24
+ A family compares a stated answer with its case. The result is the input of its scorers: a `Comparison` in `./structured`, a `LabelComparison` in `./classification`. A scorer reads that input and gives one `Ratio`.
25
+
26
+ ```ts
27
+ import { F1, Precision, Recall } from '@zukhruf/evals';
28
+ import {
29
+ ClassMatches,
30
+ LabelComparer,
31
+ LabelVarsCase,
32
+ } from '@zukhruf/evals/classification';
33
+
34
+ const comparer = new LabelComparer();
35
+ const testCase = new LabelVarsCase().from({ expected: 'program' });
36
+ const comparison = comparer.compare('home', testCase);
37
+
38
+ new Recall(new ClassMatches('program')).score(comparison); // Ratio { part: 0, whole: 1 }
39
+ new Precision(new ClassMatches('home')).score(comparison); // Ratio { part: 0, whole: 1 }
40
+ ```
41
+
42
+ A `Ratio` is a part and a whole. Its value is the part divided by the whole, and 0 when the whole is 0. `Precision`, `Recall`, `F1` and `FBeta` read the counts of a `Matches`: the hits, the stated and the expected. Each family gives its own matches, and these scorers take any of them.
43
+
44
+ ## Combine scorers
45
+
46
+ A penalty is a factor in [0, 1] for one input, as BLEU's brevity penalty is. The factor 1 is no penalty. A penalty is also a scorer: its ratio is `Ratio(factor, 1)`. A gate is the penalty whose factor is 0 or 1. A gate says what it checks (`holds`) and why it fails (`explain`).
47
+
48
+ - `new Penalized(scorer, ...penalties)` multiplies the part of the ratio by each factor. The whole stays, so a failed answer keeps its place in the denominator.
49
+ - `new AtLeast(scorer, threshold)` is a gate that holds when the value of the scorer reaches the threshold.
50
+ - `new ShortfallPenalty(scorer, { target, power })` gives `min(1, (value / target) ** power)`: 1 at the target, and less below it.
51
+ - `new SampleAveraged(scorer, { whenEmpty })` gives the value of the scorer over 1. `whenEmpty` is the value of an answer whose ratio has the whole 0.
52
+ - `new MeanOf(scorer, ...scorers)` gives the mean of the values of some scorers for one answer.
53
+
54
+ ## Score with many scorers at once
55
+
56
+ A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. The family's comparer makes the input once, and each scorer reads it.
57
+
58
+ A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core, so a `ScorerSet` holds them beside plain scorers (Inspect AI's `grouped()` applies a metric to each group in the same way).
59
+
60
+ ```ts
61
+ import { Recall, ScorerSet } from '@zukhruf/evals';
62
+ import {
63
+ AllClassMatches,
64
+ LabelComparer,
65
+ LabelsMatch,
66
+ } from '@zukhruf/evals/classification';
67
+
68
+ const pageKinds = new ScorerSet(
69
+ {
70
+ accuracy: new LabelsMatch(),
71
+ recall: new Recall(new AllClassMatches()),
72
+ },
73
+ { comparer: new LabelComparer() },
74
+ );
75
+ ```
76
+
77
+ ## Add up many answers
78
+
79
+ `Totals` adds the ratios of many scorecards, name by name: the sum of the parts divided by the sum of the wholes. So the scorer decides the average:
80
+
81
+ - A plain scorer gives its counts, such as Ratio(2, 3). `Totals` then gives the micro average.
82
+ - A `SampleAveraged` scorer gives its value over 1. `Totals` then gives the samples average: the mean of the values of the answers.
83
+
84
+ `Bootstrap` gives the interval of one name over the scorecards, with `@zukhruf/statistics`. You give it the number of resamples, the random index and the method of the interval.
85
+
86
+ ## Compare structured values
87
+
88
+ `./structured` compares a structured value with the value that a person labelled: one walk pairs the leaves of the two values, the alignment of each array is a Strategy (by position, or by best match with the Hungarian algorithm), and each scorer gives a ratio for one record. SOB's evaluate.py is a configuration of the same scorers. [docs/structured.md](docs/structured.md) shows the whole API: `ComparisonSpec`, `TreeWalk`, `Comparison`, `PathMatches`, `ValueMatches`, `ItemMatches`, `TokenF1`, `TypePrecision`, `RequiredKeyRecall`, the gates, `PerField`, `JsonSchema`, `ConfigCase` and `TextVarsCase`. Its decisions are the ADRs in [docs/adr/structured](docs/adr/structured), from the time when it was `@zukhruf/structured-eval`.
89
+
90
+ It needs two optional peers: `npm install @zukhruf/evals @cfworker/json-schema munkres`.
91
+
92
+ ## Score labels
93
+
94
+ > **Designed, not in this release.** `./classification` is built in backlog #3889 (ADR 0003). This section is its API on paper.
95
+
96
+ `./classification` compares the stated labels of an answer with its expected labels. One label is a set of one label, so one model serves a class (multiclass) and a set of classes (multilabel). The arithmetic follows scikit-learn 1.9.1, bit for bit.
97
+
98
+ ### For each answer
99
+
100
+ `LabelComparer` reads the stated labels from the answer and gives a `LabelComparison`. Its matches count one answer:
101
+
102
+ | Matches | Hits | Stated | Expected |
103
+ | ------------------------- | -------------------------------- | ------------------------ | ---------------------- |
104
+ | `new ClassMatches(label)` | 1 when both sides have the label | 1 when the answer has it | 1 when the case has it |
105
+ | `new AllClassMatches()` | the labels that both sides have | the stated labels | the expected labels |
106
+
107
+ `LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `Totals` of its ratios gives the accuracy (for a set of classes: the subset accuracy of scikit-learn). `new PerClass(scorer, { labels })` gives one ratio for each class, under the label `${class} ${label}`.
108
+
109
+ So a promptfoo run gives the micro precision, recall and F1, and the precision and recall of each class, in its column headers.
110
+
111
+ ### For a whole run
112
+
113
+ A macro or weighted average is not a ratio of sums: it takes the value of each class first. `ClassCounts` keeps the counts of each class (true positives, predicted, true), as scikit-learn's `multilabel_confusion_matrix` gives them. Add each comparison, then read the metrics:
114
+
115
+ ```ts
116
+ import {
117
+ ClassCounts,
118
+ CountsForm,
119
+ MacroAverage,
120
+ WeightedAverage,
121
+ } from '@zukhruf/evals/classification';
122
+
123
+ const counts = new ClassCounts();
124
+ for (const comparison of comparisons) counts.add(comparison);
125
+
126
+ counts.accuracy(); // accuracy_score
127
+ counts.fBeta({ beta: 1, average: new MacroAverage() }); // f1_score(average='macro')
128
+ counts.precision({ average: new WeightedAverage() }); // precision_score(average='weighted')
129
+ counts.fBeta({
130
+ beta: 0.5,
131
+ average: new MacroAverage(),
132
+ form: new CountsForm(),
133
+ });
134
+ counts.confusionMatrix(); // confusion_matrix, labels in sorted order
135
+ ```
136
+
137
+ - `average` is a Strategy: `new MicroAverage()`, `new MacroAverage()`, `new WeightedAverage()` or `new BinaryAverage(positiveLabel)`. The samples average is a `SampleAveraged` scorer, because it needs the value of each answer.
138
+ - `zeroDivision` is the value of a class score whose denominator is 0: 0 (the default, as scikit-learn's `"warn"` gives), 1 or `NaN`.
139
+ - `form` is the arithmetic of the F-measure (ADR 0003). `new CountsForm()` is scikit-learn's and the default; it also gives the bits of TREC's F0.5 for integer counts. `new PrecisionRecallForm()` is van Rijsbergen's and SOB's.
140
+ - `ClassCounts.of(stated, expected)` builds the counts from two arrays of labels, as a batch.
141
+
142
+ ### Probabilities
143
+
144
+ `BrierScore` compares the stated probability of each class with the expected labels, as scikit-learn's `brier_score_loss` does. Its ratio is the score of one answer over 1, so `Totals` gives the mean.
145
+
146
+ ## Use it with promptfoo
147
+
148
+ > In this release, `ScorerAssertion` is exported by `@zukhruf/evals/structured`, and it scores a structured value (ADR 0002). The sample below shows the API after #3889, where it moves to `./promptfoo` and also scores labels. [docs/structured.md](docs/structured.md) has the structured sample of today.
149
+
150
+ `PromptfooAssertion` adapts a scorer set to a per-row assertion. Each ratio becomes a named score with its whole as the weight, so the column header shows the same ratio of sums as `Totals`. It reads the case from the config of the assertion.
151
+
152
+ `ScorerAssertion` holds one scorer with its `weight` (how much it counts in the score of the test) and its `threshold` (when it passes). Both numbers are required. It reads the case from the vars of the test.
153
+
154
+ promptfoo stores the config of a run as JSON, and `--resume` and `--retry-errors` build the run from that stored config. A function does not survive JSON, so a config holds no scorer assertion. An `AssertionModule` holds the scorer assertions by name, in a module that promptfoo loads. Its `assert(...names)` gives JSON assertions whose `value` is the file reference `file://<module>:<name>`. promptfoo loads the function from the default export of the module when it grades, on the first run and on a resumed run (structured-eval ADR 0013).
155
+
156
+ ```ts
157
+ // page-kinds/scorers.ts: the module that promptfoo loads
158
+ import { Recall } from '@zukhruf/evals';
159
+ import {
160
+ AllClassMatches,
161
+ LabelComparer,
162
+ LabelVarsCase,
163
+ LabelsMatch,
164
+ } from '@zukhruf/evals/classification';
165
+ import { AssertionModule, ScorerAssertion } from '@zukhruf/evals/promptfoo';
166
+
167
+ const labels = { comparer: new LabelComparer(), cases: new LabelVarsCase() };
168
+
169
+ export const scorers = new AssertionModule(import.meta.url, {
170
+ accuracy: new ScorerAssertion(new LabelsMatch(), {
171
+ weight: 1,
172
+ threshold: 1,
173
+ ...labels,
174
+ }),
175
+ recall: new ScorerAssertion(new Recall(new AllClassMatches()), {
176
+ weight: 0,
177
+ threshold: 0,
178
+ ...labels,
179
+ }),
180
+ });
181
+ export default scorers.functions;
182
+ ```
183
+
184
+ ```ts
185
+ // page-kinds/promptfooconfig.ts
186
+ import type { UnifiedConfig } from 'promptfoo';
187
+
188
+ import { scorers } from './scorers.ts';
189
+
190
+ export default {
191
+ defaultTest: { assert: scorers.assert('accuracy', 'recall') },
192
+ } satisfies UnifiedConfig;
193
+ ```
194
+
195
+ The name is the promptfoo `metric`, unless the scorer assertion gives its own `metric`. A name that the module does not hold is a compile error. promptfoo shows the mean of each `metric` over the tests. A `ScorerAssertion` gives one value for one test, so a micro average needs `PromptfooAssertion`, whose ratios keep their wholes.
196
+
197
+ ## What the package does not do
198
+
199
+ - It does not run a harness, load a dataset, store results or show them. promptfoo does that.
200
+ - It has no benchmark. A benchmark, such as SOB, is a composition of scorers that its caller keeps.
201
+ - It has no LLM judge.
202
+ - A subpath never imports another subpath.
203
+
204
+ ## The same concepts in other systems
205
+
206
+ | This package | Other systems |
207
+ | -------------------------------------------------- | ---------------------------------------------------------------------------------------------- |
208
+ | Scorer (one answer), metric (many answers) | Inspect AI: `@scorer` and `@metric` |
209
+ | A subpath for each shape of answer | torchmetrics: `classification/`, `retrieval/`, `clustering/`; `@mastra/evals/scorers/prebuilt` |
210
+ | `ClassCounts`: add, then compute | torchmetrics: `update()` and `compute()` over `tp`, `fp`, `tn`, `fn` |
211
+ | `average`, `zeroDivision`, `beta` | scikit-learn: `average=`, `zero_division=`, `beta=` |
212
+ | `ScorerAssertion`: `metric`, `weight`, `threshold` | promptfoo: the fields of an assertion |
213
+ | `AssertionModule`: `file://<module>:<name>` | promptfoo: a `javascript` assertion whose `value` is a file reference |
214
+ | Optional peers of a subpath | torchmetrics extras, such as `torchmetrics[text]` |
215
+
216
+ ## API
217
+
218
+ ```ts
219
+ // @zukhruf/evals: the core
220
+
221
+ /** A part and a whole. Its value is part / whole, and 0 when the whole is 0. */
222
+ export class Ratio {
223
+ constructor(part: number, whole: number);
224
+ readonly part: number;
225
+ readonly whole: number;
226
+ get value(): number;
227
+ times(weight: number): Ratio;
228
+ }
229
+
230
+ /** Gives one ratio for one input. It keeps no result. */
231
+ export interface Scorer<Input> {
232
+ score(input: Input): Ratio;
233
+ }
234
+
235
+ /** The hits, the stated and the expected of one input. */
236
+ export class Counts {
237
+ constructor(counts: { hits: number; stated: number; expected: number });
238
+ readonly hits: number;
239
+ readonly stated: number;
240
+ readonly expected: number;
241
+ }
242
+ /** The counts of one input that precision, recall and the F-measure read. */
243
+ export interface Matches<Input> {
244
+ counts(input: Input): Counts;
245
+ }
246
+
247
+ export class Precision<Input> implements Scorer<Input> {
248
+ constructor(matches: Matches<Input>);
249
+ }
250
+ export class Recall<Input> implements Scorer<Input> {
251
+ constructor(matches: Matches<Input>);
252
+ }
253
+ /** Its part is (1 + β²) × hits and its whole is β² × expected + stated. Its value comes from its form. */
254
+ export class FBeta<Input> implements Scorer<Input> {
255
+ constructor(
256
+ matches: Matches<Input>,
257
+ options: { beta: number; form?: FMeasureForm },
258
+ );
259
+ }
260
+ /** FBeta with β = 1: its part is 2 × hits and its whole is expected + stated. */
261
+ export class F1<Input> implements Scorer<Input> {
262
+ constructor(matches: Matches<Input>, options?: { form?: FMeasureForm });
263
+ }
264
+
265
+ /** The arithmetic of the value of an F-measure (ADR 0003). */
266
+ export interface FMeasureForm {
267
+ of(counts: Counts, beta: number): number;
268
+ }
269
+ export class CountsForm implements FMeasureForm {}
270
+ export class PrecisionRecallForm implements FMeasureForm {}
271
+
272
+ /** A factor in [0, 1] for one input; its ratio is Ratio(factor, 1) (Template Method). */
273
+ export abstract class Penalty<Input> implements Scorer<Input> {
274
+ abstract factor(input: Input): number;
275
+ }
276
+ /** The penalty whose factor is 1 when it holds, else 0. */
277
+ export abstract class Gate<Input> extends Penalty<Input> {
278
+ abstract holds(input: Input): boolean;
279
+ abstract explain(input: Input): string;
280
+ }
281
+ /** Multiplies the part of the ratio of its scorer by each factor; the whole stays (Decorator). */
282
+ export class Penalized<Input> implements Scorer<Input> {
283
+ constructor(scorer: Scorer<Input>, ...penalties: Penalty<Input>[]);
284
+ }
285
+ export class AtLeast<Input> extends Gate<Input> {
286
+ constructor(scorer: Scorer<Input>, threshold: number);
287
+ }
288
+ /** min(1, (value / target) ** power). */
289
+ export class ShortfallPenalty<Input> extends Penalty<Input> {
290
+ constructor(
291
+ scorer: Scorer<Input>,
292
+ options: { target: number; power: number },
293
+ );
294
+ }
295
+ export class SampleAveraged<Input> implements Scorer<Input> {
296
+ constructor(scorer: Scorer<Input>, options: { whenEmpty: number });
297
+ }
298
+ export class MeanOf<Input> implements Scorer<Input> {
299
+ constructor(scorer: Scorer<Input>, ...scorers: Scorer<Input>[]);
300
+ }
301
+
302
+ /** What one test gives a scorer besides the stated answer. */
303
+ export interface Weighted {
304
+ readonly weight: number;
305
+ }
306
+ /** Reads a case from the plain object that a runner holds for one test (Strategy). */
307
+ export interface CaseReader<Case extends Weighted> {
308
+ from(source: Readonly<Record<string, unknown>>): Case;
309
+ }
310
+ /** Compares a stated answer with its case, and gives the input of the scorers of a family. */
311
+ export interface Comparer<Case extends Weighted, Input> {
312
+ compare(stated: unknown, testCase: Case): Input;
313
+ }
314
+
315
+ export type Check = {
316
+ readonly passed: boolean;
317
+ readonly score: number;
318
+ readonly reason: string;
319
+ };
320
+ export type Scorecard = {
321
+ readonly passed: boolean;
322
+ readonly score: number;
323
+ readonly reason: string;
324
+ readonly checks: readonly Check[];
325
+ readonly ratios: ReadonlyMap<string, Ratio>;
326
+ };
327
+ /** Scores one stated answer against its case and gives a scorecard. */
328
+ export interface ScorecardScorer<Case extends Weighted> {
329
+ score(stated: unknown, testCase: Case): Scorecard;
330
+ }
331
+ /** Gives one ratio for each group of an input, such as each field or each class. */
332
+ export abstract class Grouped<Input> {
333
+ abstract ratios(input: Input, label: string): [string, Ratio][];
334
+ }
335
+ export class ScorerSet<
336
+ Case extends Weighted,
337
+ Input,
338
+ > implements ScorecardScorer<Case> {
339
+ constructor(
340
+ scorers: Readonly<Record<string, Scorer<Input> | Grouped<Input>>>,
341
+ options: { comparer: Comparer<Case, Input> },
342
+ );
343
+ }
344
+
345
+ /** Adds ratios: the sum of the parts divided by the sum of the wholes. */
346
+ export class RatioOfSums {
347
+ add(ratio: Ratio): void;
348
+ get ratio(): Ratio;
349
+ get value(): number;
350
+ }
351
+ /** The ratio of sums of each name over many scorecards. */
352
+ export class Totals {
353
+ add(scorecard: Scorecard, weight: number): void;
354
+ get(name: string): RatioOfSums;
355
+ names(): IterableIterator<string>;
356
+ }
357
+ /** The interval of one name over weighted scorecards (@zukhruf/statistics). */
358
+ export class Bootstrap {
359
+ constructor(options: {
360
+ nResamples: number;
361
+ randomIndex: (n: number) => number;
362
+ interval: ConfidenceIntervalMethod;
363
+ });
364
+ interval(
365
+ rows: readonly { scorecard: Scorecard; weight: number }[],
366
+ name: string,
367
+ ): ConfidenceInterval;
368
+ }
369
+
370
+ // @zukhruf/evals/promptfoo
371
+
372
+ export class PromptfooAssertion<Case extends Weighted> {
373
+ constructor(scorer: ScorecardScorer<Case>, cases: CaseReader<Case>);
374
+ readonly grade: (
375
+ output: unknown,
376
+ context: { config?: Readonly<Record<string, unknown>> },
377
+ ) => GradingResult;
378
+ }
379
+ /** One scorer with its weight and threshold. It is not a promptfoo assertion: an AssertionModule gives it to promptfoo. */
380
+ export class ScorerAssertion<Case extends Weighted, Input> {
381
+ constructor(
382
+ scorer: Scorer<Input>,
383
+ options: {
384
+ weight: number;
385
+ threshold: number;
386
+ metric?: string;
387
+ comparer: Comparer<Case, Input>;
388
+ cases: CaseReader<Case>;
389
+ },
390
+ );
391
+ readonly metric: string | undefined;
392
+ readonly weight: number;
393
+ readonly threshold: number;
394
+ readonly value: (
395
+ output: string,
396
+ context: { vars: Readonly<Record<string, unknown>> },
397
+ ) => number;
398
+ }
399
+ /** A promptfoo javascript assertion that holds only JSON. */
400
+ export type FileAssertion = {
401
+ readonly type: 'javascript';
402
+ readonly metric: string;
403
+ readonly weight: number;
404
+ readonly threshold: number;
405
+ readonly value: string; // file://<module>:<name>
406
+ };
407
+ /** The module that promptfoo loads the function of each scorer assertion from (structured-eval ADR 0013). */
408
+ export class AssertionModule<Name extends string> {
409
+ constructor(
410
+ url: string,
411
+ assertions: Readonly<Record<Name, ScorerAssertion<Weighted, unknown>>>,
412
+ );
413
+ /** The default export of the module: the value function of each name. */
414
+ readonly functions: Readonly<
415
+ Record<string, ScorerAssertion<Weighted, unknown>['value']>
416
+ >;
417
+ assert(...names: Name[]): FileAssertion[];
418
+ }
419
+
420
+ // @zukhruf/evals/classification (designed; built in #3889)
421
+
422
+ /** A label: an int or a string, as scikit-learn reads a label. */
423
+ export type Label = number | string;
424
+
425
+ /** The expected labels of one test, and its weight. */
426
+ export class LabelCase implements Weighted {
427
+ constructor(expected: Label | readonly Label[], weight?: number);
428
+ readonly expected: ReadonlySet<Label>;
429
+ readonly weight: number;
430
+ }
431
+ /** Reads `vars.expected` (a label or a list of labels) and `vars.weight`. */
432
+ export class LabelVarsCase implements CaseReader<LabelCase> {}
433
+ /** Reads `config.expected`. */
434
+ export class LabelConfigCase implements CaseReader<LabelCase> {}
435
+
436
+ export class LabelComparison {
437
+ readonly stated: ReadonlySet<Label>;
438
+ readonly expected: ReadonlySet<Label>;
439
+ }
440
+ /** Reads the stated labels of an answer: a label, a list of labels, or their JSON text. */
441
+ export class LabelComparer implements Comparer<LabelCase, LabelComparison> {
442
+ constructor(options?: { normalize?: (label: string) => string });
443
+ }
444
+
445
+ export class ClassMatches implements Matches<LabelComparison> {
446
+ constructor(label: Label);
447
+ }
448
+ export class AllClassMatches implements Matches<LabelComparison> {}
449
+ /** Holds when the stated labels are the expected labels. */
450
+ export class LabelsMatch extends Gate<LabelComparison> {}
451
+ export class PerClass extends Grouped<LabelComparison> {
452
+ constructor(
453
+ scorer: Scorer<LabelComparison>,
454
+ options: { labels: readonly Label[] },
455
+ );
456
+ }
457
+
458
+ /** How the scores of the classes become one number (Strategy). */
459
+ export interface Average {}
460
+ export class MicroAverage implements Average {}
461
+ export class MacroAverage implements Average {}
462
+ export class WeightedAverage implements Average {}
463
+ export class BinaryAverage implements Average {
464
+ constructor(positiveLabel: Label);
465
+ }
466
+
467
+ /** The counts of each class over many answers: true positives, predicted and true. */
468
+ export class ClassCounts {
469
+ static of(
470
+ stated: readonly (Label | readonly Label[])[],
471
+ expected: readonly (Label | readonly Label[])[],
472
+ ): ClassCounts;
473
+ add(comparison: LabelComparison): void;
474
+ labels(): Label[];
475
+ accuracy(): number;
476
+ precision(options: {
477
+ average: Average;
478
+ zeroDivision?: number;
479
+ labels?: readonly Label[];
480
+ }): number;
481
+ recall(options: {
482
+ average: Average;
483
+ zeroDivision?: number;
484
+ labels?: readonly Label[];
485
+ }): number;
486
+ fBeta(options: {
487
+ beta: number;
488
+ average: Average;
489
+ zeroDivision?: number;
490
+ labels?: readonly Label[];
491
+ form?: FMeasureForm;
492
+ }): number;
493
+ support(): ReadonlyMap<Label, number>;
494
+ confusionMatrix(): ConfusionMatrix;
495
+ }
496
+ export class ConfusionMatrix {
497
+ readonly labels: readonly Label[];
498
+ count(expected: Label, stated: Label): number;
499
+ rows(): number[][];
500
+ }
501
+
502
+ /** The stated probability of each class of one answer. */
503
+ export class ProbabilityComparison {
504
+ readonly probabilities: ReadonlyMap<Label, number>;
505
+ readonly expected: ReadonlySet<Label>;
506
+ }
507
+ export class BrierScore implements Scorer<ProbabilityComparison> {}
508
+ ```
509
+
510
+ ## Who uses each part
511
+
512
+ The consumers below come from the search of 2026-11-06 (eight evals in four repositories). Each consumer changes to follow the package in its own backlog item.
513
+
514
+ | Consumer | Uses | Backlog |
515
+ | -------------------------------------------- | -------------------------------------------------------------------------------------------------- | ----------------------------- |
516
+ | offers-evals (SOB and the offers eval) | core, `./structured`, `./promptfoo` | offers-evals #3892 |
517
+ | context-compiler, startup-credits offers | core, `./structured` (`ByBestMatch`), `./promptfoo` | context-compiler #3893, #3882 |
518
+ | context-compiler, startup-credits page kinds | `./classification` (accuracy, the precision and recall of each class, the confusion matrix) | #3882 slice 2 |
519
+ | `@zukhruf/mteb`, classification tasks | `./classification` (`ClassCounts`: accuracy, macro and weighted P, R and F1; label-indicator rows) | mutex #3890 |
520
+ | relevance-filtering eval | `./classification` (`BinaryAverage`, F0.5 with `CountsForm`, `BrierScore`) | mutex #3891 |
521
+ | event-detection eval | nothing yet: its MUC, B³, CEAF-e and BLANC wait for `./clustering` | — |
522
+
523
+ The build is in two items: mutex #3888 (the core, `./promptfoo` and the move of `./structured`) and mutex #3889 (`./classification`).
524
+
525
+ ## Documentation
526
+
527
+ - [ADR 0001: one package, with a subpath for each shape of answer](docs/adr/0001-one-package-with-a-subpath-for-each-shape-of-answer.md)
528
+ - [ADR 0002: the core is generic over the answer](docs/adr/0002-the-core-is-generic-over-the-answer.md)
529
+ - [ADR 0003: classification follows scikit-learn, from the counts of each class](docs/adr/0003-classification-follows-scikit-learn-from-the-counts-of-each-class.md)
@@ -0,0 +1,11 @@
1
+ export declare class Counts {
2
+ readonly hits: number;
3
+ readonly stated: number;
4
+ readonly expected: number;
5
+ constructor({ hits, stated, expected, }: {
6
+ readonly hits: number;
7
+ readonly stated: number;
8
+ readonly expected: number;
9
+ });
10
+ }
11
+ //# sourceMappingURL=counts.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"counts.d.ts","sourceRoot":"","sources":["../../src/aggregation/counts.ts"],"names":[],"mappings":"AAGA,qBAAa,MAAM;IACjB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAEd,EACV,IAAI,EACJ,MAAM,EACN,QAAQ,GACT,EAAE;QACD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;KAC3B;CAKF"}
@@ -0,0 +1,14 @@
1
+ // The counts of one comparison that precision, recall and F1 read: the hits,
2
+ // the stated values and the expected values (scikit-learn's tp, tp + fp and
3
+ // tp + fn).
4
+ export class Counts {
5
+ hits;
6
+ stated;
7
+ expected;
8
+ constructor({ hits, stated, expected, }) {
9
+ this.hits = hits;
10
+ this.stated = stated;
11
+ this.expected = expected;
12
+ }
13
+ }
14
+ //# sourceMappingURL=counts.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"counts.js","sourceRoot":"","sources":["../../src/aggregation/counts.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,4EAA4E;AAC5E,YAAY;AACZ,MAAM,OAAO,MAAM;IACR,IAAI,CAAS;IACb,MAAM,CAAS;IACf,QAAQ,CAAS;IAE1B,YAAY,EACV,IAAI,EACJ,MAAM,EACN,QAAQ,GAKT;QACC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;CACF"}