@zukhruf/evals 0.0.0-stage → 0.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (292) hide show
  1. package/LICENSE +21 -0
  2. package/NOTICE +82 -0
  3. package/README.md +528 -2
  4. package/dist/aggregation/counts.d.ts +11 -0
  5. package/dist/aggregation/counts.d.ts.map +1 -0
  6. package/dist/aggregation/counts.js +14 -0
  7. package/dist/aggregation/counts.js.map +1 -0
  8. package/dist/aggregation/f1-ratio.d.ts +8 -0
  9. package/dist/aggregation/f1-ratio.d.ts.map +1 -0
  10. package/dist/aggregation/f1-ratio.js +24 -0
  11. package/dist/aggregation/f1-ratio.js.map +1 -0
  12. package/dist/aggregation/ratio-of-sums.d.ts +8 -0
  13. package/dist/aggregation/ratio-of-sums.d.ts.map +1 -0
  14. package/dist/aggregation/ratio-of-sums.js +21 -0
  15. package/dist/aggregation/ratio-of-sums.js.map +1 -0
  16. package/dist/aggregation/ratio.d.ts +8 -0
  17. package/dist/aggregation/ratio.d.ts.map +1 -0
  18. package/dist/aggregation/ratio.js +20 -0
  19. package/dist/aggregation/ratio.js.map +1 -0
  20. package/dist/index.d.ts +21 -0
  21. package/dist/index.d.ts.map +1 -0
  22. package/dist/index.js +17 -0
  23. package/dist/index.js.map +1 -0
  24. package/dist/promptfoo/assertion-module.d.ts +23 -0
  25. package/dist/promptfoo/assertion-module.d.ts.map +1 -0
  26. package/dist/promptfoo/assertion-module.js +45 -0
  27. package/dist/promptfoo/assertion-module.js.map +1 -0
  28. package/dist/promptfoo/index.d.ts +3 -0
  29. package/dist/promptfoo/index.d.ts.map +1 -0
  30. package/dist/promptfoo/index.js +3 -0
  31. package/dist/promptfoo/index.js.map +1 -0
  32. package/dist/promptfoo/promptfoo-assertion.d.ts +18 -0
  33. package/dist/promptfoo/promptfoo-assertion.d.ts.map +1 -0
  34. package/dist/promptfoo/promptfoo-assertion.js +31 -0
  35. package/dist/promptfoo/promptfoo-assertion.js.map +1 -0
  36. package/dist/scorers/at-least.d.ts +9 -0
  37. package/dist/scorers/at-least.d.ts.map +1 -0
  38. package/dist/scorers/at-least.js +18 -0
  39. package/dist/scorers/at-least.js.map +1 -0
  40. package/dist/scorers/f1.d.ts +9 -0
  41. package/dist/scorers/f1.d.ts.map +1 -0
  42. package/dist/scorers/f1.js +12 -0
  43. package/dist/scorers/f1.js.map +1 -0
  44. package/dist/scorers/gate.d.ts +7 -0
  45. package/dist/scorers/gate.d.ts.map +1 -0
  46. package/dist/scorers/gate.js +10 -0
  47. package/dist/scorers/gate.js.map +1 -0
  48. package/dist/scorers/matches.d.ts +5 -0
  49. package/dist/scorers/matches.d.ts.map +1 -0
  50. package/dist/scorers/matches.js +2 -0
  51. package/dist/scorers/matches.js.map +1 -0
  52. package/dist/scorers/mean-of.d.ts +8 -0
  53. package/dist/scorers/mean-of.d.ts.map +1 -0
  54. package/dist/scorers/mean-of.js +15 -0
  55. package/dist/scorers/mean-of.js.map +1 -0
  56. package/dist/scorers/penalized.d.ts +9 -0
  57. package/dist/scorers/penalized.d.ts.map +1 -0
  58. package/dist/scorers/penalized.js +28 -0
  59. package/dist/scorers/penalized.js.map +1 -0
  60. package/dist/scorers/penalty.d.ts +7 -0
  61. package/dist/scorers/penalty.d.ts.map +1 -0
  62. package/dist/scorers/penalty.js +11 -0
  63. package/dist/scorers/penalty.js.map +1 -0
  64. package/dist/scorers/precision.d.ts +9 -0
  65. package/dist/scorers/precision.d.ts.map +1 -0
  66. package/dist/scorers/precision.js +13 -0
  67. package/dist/scorers/precision.js.map +1 -0
  68. package/dist/scorers/recall.d.ts +9 -0
  69. package/dist/scorers/recall.d.ts.map +1 -0
  70. package/dist/scorers/recall.js +13 -0
  71. package/dist/scorers/recall.js.map +1 -0
  72. package/dist/scorers/sample-averaged.d.ts +10 -0
  73. package/dist/scorers/sample-averaged.d.ts.map +1 -0
  74. package/dist/scorers/sample-averaged.js +17 -0
  75. package/dist/scorers/sample-averaged.js.map +1 -0
  76. package/dist/scorers/scorer.d.ts +5 -0
  77. package/dist/scorers/scorer.d.ts.map +1 -0
  78. package/dist/scorers/scorer.js +2 -0
  79. package/dist/scorers/scorer.js.map +1 -0
  80. package/dist/scorers/shortfall-penalty.d.ts +11 -0
  81. package/dist/scorers/shortfall-penalty.d.ts.map +1 -0
  82. package/dist/scorers/shortfall-penalty.js +25 -0
  83. package/dist/scorers/shortfall-penalty.js.map +1 -0
  84. package/dist/scoring/bootstrap.d.ts +16 -0
  85. package/dist/scoring/bootstrap.d.ts.map +1 -0
  86. package/dist/scoring/bootstrap.js +31 -0
  87. package/dist/scoring/bootstrap.js.map +1 -0
  88. package/dist/scoring/scorecard-scorer.d.ts +11 -0
  89. package/dist/scoring/scorecard-scorer.d.ts.map +1 -0
  90. package/dist/scoring/scorecard-scorer.js +2 -0
  91. package/dist/scoring/scorecard-scorer.js.map +1 -0
  92. package/dist/scoring/scorecard.d.ts +15 -0
  93. package/dist/scoring/scorecard.d.ts.map +1 -0
  94. package/dist/scoring/scorecard.js +2 -0
  95. package/dist/scoring/scorecard.js.map +1 -0
  96. package/dist/scoring/totals.d.ts +9 -0
  97. package/dist/scoring/totals.d.ts.map +1 -0
  98. package/dist/scoring/totals.js +23 -0
  99. package/dist/scoring/totals.js.map +1 -0
  100. package/dist/structured/comparison/alignment.d.ts +9 -0
  101. package/dist/structured/comparison/alignment.d.ts.map +1 -0
  102. package/dist/structured/comparison/alignment.js +2 -0
  103. package/dist/structured/comparison/alignment.js.map +1 -0
  104. package/dist/structured/comparison/alignments/by-best-match.d.ts +9 -0
  105. package/dist/structured/comparison/alignments/by-best-match.d.ts.map +1 -0
  106. package/dist/structured/comparison/alignments/by-best-match.js +25 -0
  107. package/dist/structured/comparison/alignments/by-best-match.js.map +1 -0
  108. package/dist/structured/comparison/alignments/by-position.d.ts +6 -0
  109. package/dist/structured/comparison/alignments/by-position.d.ts.map +1 -0
  110. package/dist/structured/comparison/alignments/by-position.js +9 -0
  111. package/dist/structured/comparison/alignments/by-position.js.map +1 -0
  112. package/dist/structured/comparison/bag-of-words.d.ts +10 -0
  113. package/dist/structured/comparison/bag-of-words.d.ts.map +1 -0
  114. package/dist/structured/comparison/bag-of-words.js +48 -0
  115. package/dist/structured/comparison/bag-of-words.js.map +1 -0
  116. package/dist/structured/comparison/comparator.d.ts +5 -0
  117. package/dist/structured/comparison/comparator.d.ts.map +1 -0
  118. package/dist/structured/comparison/comparator.js +2 -0
  119. package/dist/structured/comparison/comparator.js.map +1 -0
  120. package/dist/structured/comparison/comparators/json-equality.d.ts +7 -0
  121. package/dist/structured/comparison/comparators/json-equality.d.ts.map +1 -0
  122. package/dist/structured/comparison/comparators/json-equality.js +18 -0
  123. package/dist/structured/comparison/comparators/json-equality.js.map +1 -0
  124. package/dist/structured/comparison/comparators/normalized-text.d.ts +7 -0
  125. package/dist/structured/comparison/comparators/normalized-text.d.ts.map +1 -0
  126. package/dist/structured/comparison/comparators/normalized-text.js +13 -0
  127. package/dist/structured/comparison/comparators/normalized-text.js.map +1 -0
  128. package/dist/structured/comparison/comparators/normalized.d.ts +2 -0
  129. package/dist/structured/comparison/comparators/normalized.d.ts.map +1 -0
  130. package/dist/structured/comparison/comparators/normalized.js +3 -0
  131. package/dist/structured/comparison/comparators/normalized.js.map +1 -0
  132. package/dist/structured/comparison/comparators/number-within.d.ts +9 -0
  133. package/dist/structured/comparison/comparators/number-within.d.ts.map +1 -0
  134. package/dist/structured/comparison/comparators/number-within.js +18 -0
  135. package/dist/structured/comparison/comparators/number-within.js.map +1 -0
  136. package/dist/structured/comparison/comparators/python-equality.d.ts +7 -0
  137. package/dist/structured/comparison/comparators/python-equality.d.ts.map +1 -0
  138. package/dist/structured/comparison/comparators/python-equality.js +15 -0
  139. package/dist/structured/comparison/comparators/python-equality.js.map +1 -0
  140. package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts +9 -0
  141. package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts.map +1 -0
  142. package/dist/structured/comparison/comparators/token-overlap-at-least.js +20 -0
  143. package/dist/structured/comparison/comparators/token-overlap-at-least.js.map +1 -0
  144. package/dist/structured/comparison/comparators/typed-comparator.d.ts +8 -0
  145. package/dist/structured/comparison/comparators/typed-comparator.d.ts.map +1 -0
  146. package/dist/structured/comparison/comparators/typed-comparator.js +10 -0
  147. package/dist/structured/comparison/comparators/typed-comparator.js.map +1 -0
  148. package/dist/structured/comparison/comparison-spec.d.ts +17 -0
  149. package/dist/structured/comparison/comparison-spec.d.ts.map +1 -0
  150. package/dist/structured/comparison/comparison-spec.js +36 -0
  151. package/dist/structured/comparison/comparison-spec.js.map +1 -0
  152. package/dist/structured/comparison/comparison.d.ts +25 -0
  153. package/dist/structured/comparison/comparison.d.ts.map +1 -0
  154. package/dist/structured/comparison/comparison.js +94 -0
  155. package/dist/structured/comparison/comparison.js.map +1 -0
  156. package/dist/structured/comparison/field.d.ts +10 -0
  157. package/dist/structured/comparison/field.d.ts.map +1 -0
  158. package/dist/structured/comparison/field.js +29 -0
  159. package/dist/structured/comparison/field.js.map +1 -0
  160. package/dist/structured/comparison/item-pairs.d.ts +8 -0
  161. package/dist/structured/comparison/item-pairs.d.ts.map +1 -0
  162. package/dist/structured/comparison/item-pairs.js +2 -0
  163. package/dist/structured/comparison/item-pairs.js.map +1 -0
  164. package/dist/structured/comparison/json-path-query.d.ts +7 -0
  165. package/dist/structured/comparison/json-path-query.d.ts.map +1 -0
  166. package/dist/structured/comparison/json-path-query.js +133 -0
  167. package/dist/structured/comparison/json-path-query.js.map +1 -0
  168. package/dist/structured/comparison/json-value.d.ts +10 -0
  169. package/dist/structured/comparison/json-value.d.ts.map +1 -0
  170. package/dist/structured/comparison/json-value.js +47 -0
  171. package/dist/structured/comparison/json-value.js.map +1 -0
  172. package/dist/structured/comparison/leaf-pair.d.ts +24 -0
  173. package/dist/structured/comparison/leaf-pair.d.ts.map +1 -0
  174. package/dist/structured/comparison/leaf-pair.js +2 -0
  175. package/dist/structured/comparison/leaf-pair.js.map +1 -0
  176. package/dist/structured/comparison/match.d.ts +5 -0
  177. package/dist/structured/comparison/match.d.ts.map +1 -0
  178. package/dist/structured/comparison/match.js +2 -0
  179. package/dist/structured/comparison/match.js.map +1 -0
  180. package/dist/structured/comparison/name-selector.d.ts +6 -0
  181. package/dist/structured/comparison/name-selector.d.ts.map +1 -0
  182. package/dist/structured/comparison/name-selector.js +41 -0
  183. package/dist/structured/comparison/name-selector.js.map +1 -0
  184. package/dist/structured/comparison/path.d.ts +10 -0
  185. package/dist/structured/comparison/path.d.ts.map +1 -0
  186. package/dist/structured/comparison/path.js +27 -0
  187. package/dist/structured/comparison/path.js.map +1 -0
  188. package/dist/structured/comparison/tree-walk.d.ts +10 -0
  189. package/dist/structured/comparison/tree-walk.d.ts.map +1 -0
  190. package/dist/structured/comparison/tree-walk.js +136 -0
  191. package/dist/structured/comparison/tree-walk.js.map +1 -0
  192. package/dist/structured/cpython/python-text.d.ts +5 -0
  193. package/dist/structured/cpython/python-text.d.ts.map +1 -0
  194. package/dist/structured/cpython/python-text.js +79 -0
  195. package/dist/structured/cpython/python-text.js.map +1 -0
  196. package/dist/structured/index.d.ts +39 -0
  197. package/dist/structured/index.d.ts.map +1 -0
  198. package/dist/structured/index.js +29 -0
  199. package/dist/structured/index.js.map +1 -0
  200. package/dist/structured/promptfoo/scorer-assertion.d.ts +22 -0
  201. package/dist/structured/promptfoo/scorer-assertion.d.ts.map +1 -0
  202. package/dist/structured/promptfoo/scorer-assertion.js +34 -0
  203. package/dist/structured/promptfoo/scorer-assertion.js.map +1 -0
  204. package/dist/structured/schema/draft-07-schema.d.ts +3 -0
  205. package/dist/structured/schema/draft-07-schema.d.ts.map +1 -0
  206. package/dist/structured/schema/draft-07-schema.js +243 -0
  207. package/dist/structured/schema/draft-07-schema.js.map +1 -0
  208. package/dist/structured/schema/draft-2020-12-schema.d.ts +4 -0
  209. package/dist/structured/schema/draft-2020-12-schema.d.ts.map +1 -0
  210. package/dist/structured/schema/draft-2020-12-schema.js +280 -0
  211. package/dist/structured/schema/draft-2020-12-schema.js.map +1 -0
  212. package/dist/structured/schema/is-record.d.ts +2 -0
  213. package/dist/structured/schema/is-record.d.ts.map +1 -0
  214. package/dist/structured/schema/is-record.js +3 -0
  215. package/dist/structured/schema/is-record.js.map +1 -0
  216. package/dist/structured/schema/json-schema.d.ts +20 -0
  217. package/dist/structured/schema/json-schema.d.ts.map +1 -0
  218. package/dist/structured/schema/json-schema.js +244 -0
  219. package/dist/structured/schema/json-schema.js.map +1 -0
  220. package/dist/structured/schema/schema-type-walk.d.ts +11 -0
  221. package/dist/structured/schema/schema-type-walk.d.ts.map +1 -0
  222. package/dist/structured/schema/schema-type-walk.js +196 -0
  223. package/dist/structured/schema/schema-type-walk.js.map +1 -0
  224. package/dist/structured/scorers/all-matched.d.ts +10 -0
  225. package/dist/structured/scorers/all-matched.d.ts.map +1 -0
  226. package/dist/structured/scorers/all-matched.js +36 -0
  227. package/dist/structured/scorers/all-matched.js.map +1 -0
  228. package/dist/structured/scorers/exact-match.d.ts +7 -0
  229. package/dist/structured/scorers/exact-match.d.ts.map +1 -0
  230. package/dist/structured/scorers/exact-match.js +13 -0
  231. package/dist/structured/scorers/exact-match.js.map +1 -0
  232. package/dist/structured/scorers/item-matches.d.ts +7 -0
  233. package/dist/structured/scorers/item-matches.d.ts.map +1 -0
  234. package/dist/structured/scorers/item-matches.js +17 -0
  235. package/dist/structured/scorers/item-matches.js.map +1 -0
  236. package/dist/structured/scorers/leaf-matches.d.ts +14 -0
  237. package/dist/structured/scorers/leaf-matches.d.ts.map +1 -0
  238. package/dist/structured/scorers/leaf-matches.js +28 -0
  239. package/dist/structured/scorers/leaf-matches.js.map +1 -0
  240. package/dist/structured/scorers/parse-success.d.ts +7 -0
  241. package/dist/structured/scorers/parse-success.d.ts.map +1 -0
  242. package/dist/structured/scorers/parse-success.js +11 -0
  243. package/dist/structured/scorers/parse-success.js.map +1 -0
  244. package/dist/structured/scorers/path-matches.d.ts +5 -0
  245. package/dist/structured/scorers/path-matches.d.ts.map +1 -0
  246. package/dist/structured/scorers/path-matches.js +9 -0
  247. package/dist/structured/scorers/path-matches.js.map +1 -0
  248. package/dist/structured/scorers/required-key-recall.d.ts +8 -0
  249. package/dist/structured/scorers/required-key-recall.d.ts.map +1 -0
  250. package/dist/structured/scorers/required-key-recall.js +50 -0
  251. package/dist/structured/scorers/required-key-recall.js.map +1 -0
  252. package/dist/structured/scorers/schema-compliance.d.ts +8 -0
  253. package/dist/structured/scorers/schema-compliance.d.ts.map +1 -0
  254. package/dist/structured/scorers/schema-compliance.js +29 -0
  255. package/dist/structured/scorers/schema-compliance.js.map +1 -0
  256. package/dist/structured/scorers/token-f1.d.ts +14 -0
  257. package/dist/structured/scorers/token-f1.d.ts.map +1 -0
  258. package/dist/structured/scorers/token-f1.js +44 -0
  259. package/dist/structured/scorers/token-f1.js.map +1 -0
  260. package/dist/structured/scorers/type-precision.d.ts +10 -0
  261. package/dist/structured/scorers/type-precision.d.ts.map +1 -0
  262. package/dist/structured/scorers/type-precision.js +30 -0
  263. package/dist/structured/scorers/type-precision.js.map +1 -0
  264. package/dist/structured/scorers/valid-schema.d.ts +7 -0
  265. package/dist/structured/scorers/valid-schema.d.ts.map +1 -0
  266. package/dist/structured/scorers/valid-schema.js +11 -0
  267. package/dist/structured/scorers/valid-schema.js.map +1 -0
  268. package/dist/structured/scorers/value-matches.d.ts +6 -0
  269. package/dist/structured/scorers/value-matches.d.ts.map +1 -0
  270. package/dist/structured/scorers/value-matches.js +9 -0
  271. package/dist/structured/scorers/value-matches.js.map +1 -0
  272. package/dist/structured/scoring/case.d.ts +14 -0
  273. package/dist/structured/scoring/case.d.ts.map +1 -0
  274. package/dist/structured/scoring/case.js +14 -0
  275. package/dist/structured/scoring/case.js.map +1 -0
  276. package/dist/structured/scoring/config-case.d.ts +11 -0
  277. package/dist/structured/scoring/config-case.d.ts.map +1 -0
  278. package/dist/structured/scoring/config-case.js +22 -0
  279. package/dist/structured/scoring/config-case.js.map +1 -0
  280. package/dist/structured/scoring/per-field.d.ts +9 -0
  281. package/dist/structured/scoring/per-field.d.ts.map +1 -0
  282. package/dist/structured/scoring/per-field.js +23 -0
  283. package/dist/structured/scoring/per-field.js.map +1 -0
  284. package/dist/structured/scoring/scorer-set.d.ts +15 -0
  285. package/dist/structured/scoring/scorer-set.d.ts.map +1 -0
  286. package/dist/structured/scoring/scorer-set.js +60 -0
  287. package/dist/structured/scoring/scorer-set.js.map +1 -0
  288. package/dist/structured/scoring/text-vars-case.d.ts +12 -0
  289. package/dist/structured/scoring/text-vars-case.d.ts.map +1 -0
  290. package/dist/structured/scoring/text-vars-case.js +31 -0
  291. package/dist/structured/scoring/text-vars-case.js.map +1 -0
  292. package/package.json +78 -4
@@ -0,0 +1,8 @@
1
+ import type { Counts } from './counts.ts';
2
+ import { Ratio } from './ratio.ts';
3
+ export declare class F1Ratio extends Ratio {
4
+ #private;
5
+ constructor(counts: Counts);
6
+ get value(): number;
7
+ }
8
+ //# sourceMappingURL=f1-ratio.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"f1-ratio.d.ts","sourceRoot":"","sources":["../../src/aggregation/f1-ratio.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AASnC,qBAAa,OAAQ,SAAQ,KAAK;;gBAGpB,MAAM,EAAE,MAAM;IAK1B,IAAa,KAAK,IAAI,MAAM,CAM3B;CACF"}
@@ -0,0 +1,24 @@
1
+ import { Ratio } from "./ratio.js";
2
+ // The ratio of an F1: its part is 2 × hits and its whole is stated +
3
+ // expected, so many F1 ratios add up to the micro F1. Its value is 0 when
4
+ // there are no hits, else 2·p·r/(p + r): precision and recall first, the
5
+ // arithmetic of SOB's evaluate.py (f1_from_counts, token_f1). That value can
6
+ // differ from part / whole in the last bit, such as for 1 hit of 1 expected
7
+ // and 5 stated (0.33333333333333337, not 1/3). It is the one exception to the
8
+ // rule of a Ratio (ADR 0011).
9
+ export class F1Ratio extends Ratio {
10
+ #counts;
11
+ constructor(counts) {
12
+ super(2 * counts.hits, counts.stated + counts.expected);
13
+ this.#counts = counts;
14
+ }
15
+ get value() {
16
+ const { hits, stated, expected } = this.#counts;
17
+ if (hits === 0)
18
+ return 0;
19
+ const precision = hits / stated;
20
+ const recall = hits / expected;
21
+ return (2 * precision * recall) / (precision + recall);
22
+ }
23
+ }
24
+ //# sourceMappingURL=f1-ratio.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"f1-ratio.js","sourceRoot":"","sources":["../../src/aggregation/f1-ratio.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,qEAAqE;AACrE,0EAA0E;AAC1E,yEAAyE;AACzE,6EAA6E;AAC7E,4EAA4E;AAC5E,8EAA8E;AAC9E,8BAA8B;AAC9B,MAAM,OAAO,OAAQ,SAAQ,KAAK;IACvB,OAAO,CAAS;IAEzB,YAAY,MAAc;QACxB,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,IAAa,KAAK;QAChB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QAChD,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,GAAG,MAAM,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,GAAG,QAAQ,CAAC;QAC/B,OAAO,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IACzD,CAAC;CACF"}
@@ -0,0 +1,8 @@
1
+ import { Ratio } from './ratio.ts';
2
+ export declare class RatioOfSums {
3
+ #private;
4
+ add(ratio: Ratio): void;
5
+ get ratio(): Ratio;
6
+ get value(): number;
7
+ }
8
+ //# sourceMappingURL=ratio-of-sums.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ratio-of-sums.d.ts","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMnC,qBAAa,WAAW;;IAItB,GAAG,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAKvB,IAAI,KAAK,IAAI,KAAK,CAEjB;IAED,IAAI,KAAK,IAAI,MAAM,CAElB;CACF"}
@@ -0,0 +1,21 @@
1
+ import { NeumaierSum } from '@zukhruf/python';
2
+ import { Ratio } from "./ratio.js";
3
+ // The sum of the parts of many ratios divided by the sum of their wholes.
4
+ // Each average of the package is one (ADR 0004): a micro average adds
5
+ // Ratio(right, expected), a mean adds Ratio(value, 1), and a weighted mean
6
+ // adds Ratio(value × weight, weight).
7
+ export class RatioOfSums {
8
+ #parts = new NeumaierSum();
9
+ #wholes = new NeumaierSum();
10
+ add(ratio) {
11
+ this.#parts.add(ratio.part);
12
+ this.#wholes.add(ratio.whole);
13
+ }
14
+ get ratio() {
15
+ return new Ratio(this.#parts.value, this.#wholes.value);
16
+ }
17
+ get value() {
18
+ return this.ratio.value;
19
+ }
20
+ }
21
+ //# sourceMappingURL=ratio-of-sums.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ratio-of-sums.js","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,0EAA0E;AAC1E,sEAAsE;AACtE,2EAA2E;AAC3E,sCAAsC;AACtC,MAAM,OAAO,WAAW;IACb,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;IAC3B,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAErC,GAAG,CAAC,KAAY;QACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;IAC1B,CAAC;CACF"}
@@ -0,0 +1,8 @@
1
+ export declare class Ratio {
2
+ readonly part: number;
3
+ readonly whole: number;
4
+ constructor(part: number, whole: number);
5
+ get value(): number;
6
+ times(weight: number): Ratio;
7
+ }
8
+ //# sourceMappingURL=ratio.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ratio.d.ts","sourceRoot":"","sources":["../../src/aggregation/ratio.ts"],"names":[],"mappings":"AAEA,qBAAa,KAAK;IAChB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAEX,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAMvC,IAAI,KAAK,IAAI,MAAM,CAElB;IAID,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK;CAG7B"}
@@ -0,0 +1,20 @@
1
+ // A part and a whole, such as 2 right values of 3 expected values. It keeps
2
+ // both, not only their quotient, so many ratios add up exactly (ADR 0004).
3
+ export class Ratio {
4
+ part;
5
+ whole;
6
+ constructor(part, whole) {
7
+ this.part = part;
8
+ this.whole = whole;
9
+ }
10
+ // 0 when the whole is 0, as the ratio() of SOB's evaluate.py gives.
11
+ get value() {
12
+ return this.whole === 0 ? 0 : this.part / this.whole;
13
+ }
14
+ // Ratio(value, 1).times(weight) is Ratio(value × weight, weight): the term
15
+ // of a weighted mean.
16
+ times(weight) {
17
+ return new Ratio(this.part * weight, this.whole * weight);
18
+ }
19
+ }
20
+ //# sourceMappingURL=ratio.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ratio.js","sourceRoot":"","sources":["../../src/aggregation/ratio.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,2EAA2E;AAC3E,MAAM,OAAO,KAAK;IACP,IAAI,CAAS;IACb,KAAK,CAAS;IAEvB,YAAY,IAAY,EAAE,KAAa;QACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,oEAAoE;IACpE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC;IACvD,CAAC;IAED,2EAA2E;IAC3E,sBAAsB;IACtB,KAAK,CAAC,MAAc;QAClB,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;IAC5D,CAAC;CACF"}
@@ -0,0 +1,21 @@
1
+ export { Ratio } from './aggregation/ratio.ts';
2
+ export { RatioOfSums } from './aggregation/ratio-of-sums.ts';
3
+ export { Counts } from './aggregation/counts.ts';
4
+ export { F1Ratio } from './aggregation/f1-ratio.ts';
5
+ export type { Check, Scorecard } from './scoring/scorecard.ts';
6
+ export type { CaseReader, ScorecardScorer, Weighted, } from './scoring/scorecard-scorer.ts';
7
+ export { Totals } from './scoring/totals.ts';
8
+ export { Bootstrap, type WeightedScorecard } from './scoring/bootstrap.ts';
9
+ export type { Scorer } from './scorers/scorer.ts';
10
+ export type { Matches } from './scorers/matches.ts';
11
+ export { Precision } from './scorers/precision.ts';
12
+ export { Recall } from './scorers/recall.ts';
13
+ export { F1 } from './scorers/f1.ts';
14
+ export { Penalty } from './scorers/penalty.ts';
15
+ export { Gate } from './scorers/gate.ts';
16
+ export { Penalized } from './scorers/penalized.ts';
17
+ export { AtLeast } from './scorers/at-least.ts';
18
+ export { ShortfallPenalty } from './scorers/shortfall-penalty.ts';
19
+ export { SampleAveraged } from './scorers/sample-averaged.ts';
20
+ export { MeanOf } from './scorers/mean-of.ts';
21
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAC;AACpD,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAC/D,YAAY,EACV,UAAU,EACV,eAAe,EACf,QAAQ,GACT,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,KAAK,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,YAAY,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,YAAY,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC"}
package/dist/index.js ADDED
@@ -0,0 +1,17 @@
1
+ export { Ratio } from "./aggregation/ratio.js";
2
+ export { RatioOfSums } from "./aggregation/ratio-of-sums.js";
3
+ export { Counts } from "./aggregation/counts.js";
4
+ export { F1Ratio } from "./aggregation/f1-ratio.js";
5
+ export { Totals } from "./scoring/totals.js";
6
+ export { Bootstrap } from "./scoring/bootstrap.js";
7
+ export { Precision } from "./scorers/precision.js";
8
+ export { Recall } from "./scorers/recall.js";
9
+ export { F1 } from "./scorers/f1.js";
10
+ export { Penalty } from "./scorers/penalty.js";
11
+ export { Gate } from "./scorers/gate.js";
12
+ export { Penalized } from "./scorers/penalized.js";
13
+ export { AtLeast } from "./scorers/at-least.js";
14
+ export { ShortfallPenalty } from "./scorers/shortfall-penalty.js";
15
+ export { SampleAveraged } from "./scorers/sample-averaged.js";
16
+ export { MeanOf } from "./scorers/mean-of.js";
17
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAC;AAOpD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAA0B,MAAM,wBAAwB,CAAC;AAG3E,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC"}
@@ -0,0 +1,23 @@
1
+ export type VarsContext = {
2
+ readonly vars: Readonly<Record<string, unknown>>;
3
+ };
4
+ export type ModuleAssertion = {
5
+ readonly metric?: string | undefined;
6
+ readonly weight: number;
7
+ readonly threshold: number;
8
+ readonly value: (output: string, context: VarsContext) => number;
9
+ };
10
+ export type FileAssertion = {
11
+ readonly type: 'javascript';
12
+ readonly metric: string;
13
+ readonly weight: number;
14
+ readonly threshold: number;
15
+ readonly value: string;
16
+ };
17
+ export declare class AssertionModule<Name extends string> {
18
+ #private;
19
+ readonly functions: Readonly<Record<string, ModuleAssertion['value']>>;
20
+ constructor(url: string, assertions: Readonly<Record<Name, ModuleAssertion>>);
21
+ assert(...names: Name[]): FileAssertion[];
22
+ }
23
+ //# sourceMappingURL=assertion-module.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"assertion-module.d.ts","sourceRoot":"","sources":["../../src/promptfoo/assertion-module.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CAClD,CAAC;AAKF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,KAAK,MAAM,CAAC;CAClE,CAAC;AAKF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAgBF,qBAAa,eAAe,CAAC,IAAI,SAAS,MAAM;;IAG9C,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAKrE,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;IAcrD,MAAM,CAAC,GAAG,KAAK,EAAE,IAAI,EAAE,GAAG,aAAa,EAAE;CAY1C"}
@@ -0,0 +1,45 @@
1
+ import { fileURLToPath } from 'node:url';
2
+ // The module that promptfoo loads the function of each scorer assertion from
3
+ // (ADR 0013). promptfoo stores the config of an eval as JSON, and --resume
4
+ // and --retry-errors build the run from that stored config, so a function in
5
+ // the config is lost and its assertion scores 0. A config holds the file
6
+ // references that assert gives, and promptfoo loads each function from the
7
+ // default export of the module: importModule reads file://<module>:<name> on
8
+ // the default export when the module has one (promptfoo 0.125.1,
9
+ // dist/src/esm-B4NX_h4K.js:205-224). So the module gives functions as its
10
+ // default export, and url is the module's own import.meta.url.
11
+ //
12
+ // The reference holds the absolute path of the module. promptfoo 0.125.1
13
+ // splits a file reference at its first colon
14
+ // (dist/src/evaluator-BxCmvGtv.js:5683-5686), so on Windows the drive of the
15
+ // path ends the path: a defect of promptfoo.
16
+ export class AssertionModule {
17
+ // promptfoo reads a function by the name in its reference, so the names are
18
+ // keys, and assert holds the type of each name.
19
+ functions;
20
+ #path;
21
+ #assertions;
22
+ constructor(url, assertions) {
23
+ this.#path = fileURLToPath(url);
24
+ this.#assertions = assertions;
25
+ this.functions = Object.fromEntries(Object.entries(assertions).map(([name, assertion]) => [
26
+ name,
27
+ assertion.value,
28
+ ]));
29
+ }
30
+ // One assertion for each name. The name is the metric, unless its scorer
31
+ // assertion gives one.
32
+ assert(...names) {
33
+ return names.map((name) => {
34
+ const { metric = name, weight, threshold } = this.#assertions[name];
35
+ return {
36
+ type: 'javascript',
37
+ metric,
38
+ weight,
39
+ threshold,
40
+ value: `file://${this.#path}:${name}`,
41
+ };
42
+ });
43
+ }
44
+ }
45
+ //# sourceMappingURL=assertion-module.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"assertion-module.js","sourceRoot":"","sources":["../../src/promptfoo/assertion-module.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AA6BzC,6EAA6E;AAC7E,2EAA2E;AAC3E,6EAA6E;AAC7E,yEAAyE;AACzE,2EAA2E;AAC3E,6EAA6E;AAC7E,iEAAiE;AACjE,0EAA0E;AAC1E,+DAA+D;AAC/D,EAAE;AACF,yEAAyE;AACzE,6CAA6C;AAC7C,6EAA6E;AAC7E,6CAA6C;AAC7C,MAAM,OAAO,eAAe;IAC1B,4EAA4E;IAC5E,gDAAgD;IACvC,SAAS,CAAqD;IAC9D,KAAK,CAAS;IACd,WAAW,CAA0C;IAE9D,YACE,GAAW,EACX,UAAmD;QAEnD,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,WAAW,CACjC,MAAM,CAAC,OAAO,CAAkB,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC;YACrE,IAAI;YACJ,SAAS,CAAC,KAAK;SAChB,CAAC,CACH,CAAC;IACJ,CAAC;IAED,yEAAyE;IACzE,uBAAuB;IACvB,MAAM,CAAC,GAAG,KAAa;QACrB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACxB,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACpE,OAAO;gBACL,IAAI,EAAE,YAAY;gBAClB,MAAM;gBACN,MAAM;gBACN,SAAS;gBACT,KAAK,EAAE,UAAU,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE;aACtC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
@@ -0,0 +1,3 @@
1
+ export { AssertionModule, type FileAssertion, type ModuleAssertion, type VarsContext, } from './assertion-module.ts';
2
+ export { type AssertionContext, type GradingResult, PromptfooAssertion, } from './promptfoo-assertion.ts';
3
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/promptfoo/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,WAAW,GACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAClB,kBAAkB,GACnB,MAAM,0BAA0B,CAAC"}
@@ -0,0 +1,3 @@
1
+ export { AssertionModule, } from "./assertion-module.js";
2
+ export { PromptfooAssertion, } from "./promptfoo-assertion.js";
3
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/promptfoo/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,GAIhB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAGL,kBAAkB,GACnB,MAAM,0BAA0B,CAAC"}
@@ -0,0 +1,18 @@
1
+ import type { CaseReader, ScorecardScorer, Weighted } from '../scoring/scorecard-scorer.ts';
2
+ export type GradingResult = {
3
+ readonly pass: boolean;
4
+ readonly score: number;
5
+ readonly reason: string;
6
+ readonly namedScores?: Readonly<Record<string, number>>;
7
+ readonly namedScoreWeights?: Readonly<Record<string, number>>;
8
+ readonly componentResults?: readonly GradingResult[];
9
+ };
10
+ export type AssertionContext = {
11
+ readonly config?: Readonly<Record<string, unknown>>;
12
+ };
13
+ export declare class PromptfooAssertion<Case extends Weighted> {
14
+ #private;
15
+ constructor(scorer: ScorecardScorer<Case>, cases: CaseReader<Case>);
16
+ readonly grade: (output: unknown, context: AssertionContext) => GradingResult;
17
+ }
18
+ //# sourceMappingURL=promptfoo-assertion.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"promptfoo-assertion.d.ts","sourceRoot":"","sources":["../../src/promptfoo/promptfoo-assertion.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EACV,eAAe,EACf,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAKxC,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9D,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;CACtD,CAAC;AAIF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACrD,CAAC;AAOF,qBAAa,kBAAkB,CAAC,IAAI,SAAS,QAAQ;;gBAIvC,MAAM,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC;IAKlE,QAAQ,CAAC,KAAK,GACZ,QAAQ,OAAO,EACf,SAAS,gBAAgB,KACxB,aAAa,CAsBd;CACH"}
@@ -0,0 +1,31 @@
1
+ // Adapts a scorer to a promptfoo javascript assertion (ADR 0004). Each ratio
2
+ // of the scorecard becomes a named score with its whole as the weight:
3
+ // promptfoo adds value × weight over the rows and divides by the weights
4
+ // (namedMetrics.ts, accumulateNamedMetric), which is the ratio of sums.
5
+ // promptfoo calls a function, so a module exports `grade`.
6
+ export class PromptfooAssertion {
7
+ #scorer;
8
+ #cases;
9
+ constructor(scorer, cases) {
10
+ this.#scorer = scorer;
11
+ this.#cases = cases;
12
+ }
13
+ grade = (output, context) => {
14
+ const testCase = this.#cases.from(context.config ?? {});
15
+ const scorecard = this.#scorer.score(output, testCase);
16
+ const ratios = [...scorecard.ratios].map(([name, ratio]) => [name, ratio.times(testCase.weight)]);
17
+ return {
18
+ pass: scorecard.passed,
19
+ score: scorecard.score,
20
+ reason: scorecard.reason,
21
+ namedScores: Object.fromEntries(ratios.map(([name, ratio]) => [name, ratio.value])),
22
+ namedScoreWeights: Object.fromEntries(ratios.map(([name, ratio]) => [name, ratio.whole])),
23
+ componentResults: scorecard.checks.map((check) => ({
24
+ pass: check.passed,
25
+ score: check.score,
26
+ reason: check.reason,
27
+ })),
28
+ };
29
+ };
30
+ }
31
+ //# sourceMappingURL=promptfoo-assertion.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"promptfoo-assertion.js","sourceRoot":"","sources":["../../src/promptfoo/promptfoo-assertion.ts"],"names":[],"mappings":"AAwBA,6EAA6E;AAC7E,uEAAuE;AACvE,yEAAyE;AACzE,wEAAwE;AACxE,2DAA2D;AAC3D,MAAM,OAAO,kBAAkB;IACpB,OAAO,CAAwB;IAC/B,MAAM,CAAmB;IAElC,YAAY,MAA6B,EAAE,KAAuB;QAChE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAEQ,KAAK,GAAG,CACf,MAAe,EACf,OAAyB,EACV,EAAE;QACjB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,CACtC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAU,CACjE,CAAC;QACF,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,WAAW,EAAE,MAAM,CAAC,WAAW,CAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CACnD;YACD,iBAAiB,EAAE,MAAM,CAAC,WAAW,CACnC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CACnD;YACD,gBAAgB,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACjD,IAAI,EAAE,KAAK,CAAC,MAAM;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,MAAM,EAAE,KAAK,CAAC,MAAM;aACrB,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC,CAAC;CACH"}
@@ -0,0 +1,9 @@
1
+ import { Gate } from './gate.ts';
2
+ import type { Scorer } from './scorer.ts';
3
+ export declare class AtLeast<Input> extends Gate<Input> {
4
+ #private;
5
+ constructor(scorer: Scorer<Input>, threshold: number);
6
+ holds(input: Input): boolean;
7
+ explain(input: Input): string;
8
+ }
9
+ //# sourceMappingURL=at-least.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"at-least.d.ts","sourceRoot":"","sources":["../../src/scorers/at-least.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,qBAAa,OAAO,CAAC,KAAK,CAAE,SAAQ,IAAI,CAAC,KAAK,CAAC;;gBAIjC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,MAAM;IAMpD,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO;IAI5B,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;CAG9B"}
@@ -0,0 +1,18 @@
1
+ import { Gate } from "./gate.js";
2
+ // A gate: it holds when the value of the scorer reaches the threshold.
3
+ export class AtLeast extends Gate {
4
+ #scorer;
5
+ #threshold;
6
+ constructor(scorer, threshold) {
7
+ super();
8
+ this.#scorer = scorer;
9
+ this.#threshold = threshold;
10
+ }
11
+ holds(input) {
12
+ return this.#scorer.score(input).value >= this.#threshold;
13
+ }
14
+ explain(input) {
15
+ return `the value ${this.#scorer.score(input).value} is below the threshold ${this.#threshold}`;
16
+ }
17
+ }
18
+ //# sourceMappingURL=at-least.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"at-least.js","sourceRoot":"","sources":["../../src/scorers/at-least.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,uEAAuE;AACvE,MAAM,OAAO,OAAe,SAAQ,IAAW;IACpC,OAAO,CAAgB;IACvB,UAAU,CAAS;IAE5B,YAAY,MAAqB,EAAE,SAAiB;QAClD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC;IAC5D,CAAC;IAED,OAAO,CAAC,KAAY;QAClB,OAAO,aAAa,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,2BAA2B,IAAI,CAAC,UAAU,EAAE,CAAC;IAClG,CAAC;CACF"}
@@ -0,0 +1,9 @@
1
+ import { F1Ratio } from '../aggregation/f1-ratio.ts';
2
+ import type { Matches } from './matches.ts';
3
+ import type { Scorer } from './scorer.ts';
4
+ export declare class F1<Input> implements Scorer<Input> {
5
+ #private;
6
+ constructor(matches: Matches<Input>);
7
+ score(input: Input): F1Ratio;
8
+ }
9
+ //# sourceMappingURL=f1.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"f1.d.ts","sourceRoot":"","sources":["../../src/scorers/f1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,qBAAa,EAAE,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAGjC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC;IAInC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO;CAG7B"}
@@ -0,0 +1,12 @@
1
+ import { F1Ratio } from "../aggregation/f1-ratio.js";
2
+ // The F1 of the hits: an F1Ratio of the counts.
3
+ export class F1 {
4
+ #matches;
5
+ constructor(matches) {
6
+ this.#matches = matches;
7
+ }
8
+ score(input) {
9
+ return new F1Ratio(this.#matches.counts(input));
10
+ }
11
+ }
12
+ //# sourceMappingURL=f1.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"f1.js","sourceRoot":"","sources":["../../src/scorers/f1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAIrD,gDAAgD;AAChD,MAAM,OAAO,EAAE;IACJ,QAAQ,CAAiB;IAElC,YAAY,OAAuB;QACjC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAClD,CAAC;CACF"}
@@ -0,0 +1,7 @@
1
+ import { Penalty } from './penalty.ts';
2
+ export declare abstract class Gate<Input> extends Penalty<Input> {
3
+ abstract holds(input: Input): boolean;
4
+ abstract explain(input: Input): string;
5
+ factor(input: Input): number;
6
+ }
7
+ //# sourceMappingURL=gate.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"gate.d.ts","sourceRoot":"","sources":["../../src/scorers/gate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAKvC,8BAAsB,IAAI,CAAC,KAAK,CAAE,SAAQ,OAAO,CAAC,KAAK,CAAC;IACtD,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO;IAGrC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;IAEtC,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;CAG7B"}
@@ -0,0 +1,10 @@
1
+ import { Penalty } from "./penalty.js";
2
+ // The penalty whose factor is 1 when the input holds what the gate checks,
3
+ // else 0. A subclass says what it checks and why an input fails it (Template
4
+ // Method), so a ScorerSet makes a check of each gate.
5
+ export class Gate extends Penalty {
6
+ factor(input) {
7
+ return this.holds(input) ? 1 : 0;
8
+ }
9
+ }
10
+ //# sourceMappingURL=gate.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"gate.js","sourceRoot":"","sources":["../../src/scorers/gate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,2EAA2E;AAC3E,6EAA6E;AAC7E,sDAAsD;AACtD,MAAM,OAAgB,IAAY,SAAQ,OAAc;IAMtD,MAAM,CAAC,KAAY;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;CACF"}
@@ -0,0 +1,5 @@
1
+ import type { Counts } from '../aggregation/counts.ts';
2
+ export interface Matches<Input> {
3
+ counts(input: Input): Counts;
4
+ }
5
+ //# sourceMappingURL=matches.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"matches.d.ts","sourceRoot":"","sources":["../../src/scorers/matches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIvD,MAAM,WAAW,OAAO,CAAC,KAAK;IAC5B,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC;CAC9B"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=matches.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"matches.js","sourceRoot":"","sources":["../../src/scorers/matches.ts"],"names":[],"mappings":""}
@@ -0,0 +1,8 @@
1
+ import { Ratio } from '../aggregation/ratio.ts';
2
+ import type { Scorer } from './scorer.ts';
3
+ export declare class MeanOf<Input> implements Scorer<Input> {
4
+ #private;
5
+ constructor(scorer: Scorer<Input>, ...scorers: readonly Scorer<Input>[]);
6
+ score(input: Input): Ratio;
7
+ }
8
+ //# sourceMappingURL=mean-of.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"mean-of.d.ts","sourceRoot":"","sources":["../../src/scorers/mean-of.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAK1C,qBAAa,MAAM,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAGrC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,EAAE,SAAS,MAAM,CAAC,KAAK,CAAC,EAAE;IAIvE,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAQ3B"}
@@ -0,0 +1,15 @@
1
+ import { NeumaierSum } from '@zukhruf/python';
2
+ import { Ratio } from "../aggregation/ratio.js";
3
+ // The mean of the values of some scorers for one record, over 1, with
4
+ // CPython's sum, such as a category of a benchmark. It takes one scorer or
5
+ // more, because the mean of no value has no value.
6
+ export class MeanOf {
7
+ #scorers;
8
+ constructor(scorer, ...scorers) {
9
+ this.#scorers = [scorer, ...scorers];
10
+ }
11
+ score(input) {
12
+ return new Ratio(NeumaierSum.mean(this.#scorers.map((scorer) => scorer.score(input).value)), 1);
13
+ }
14
+ }
15
+ //# sourceMappingURL=mean-of.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"mean-of.js","sourceRoot":"","sources":["../../src/scorers/mean-of.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGhD,sEAAsE;AACtE,2EAA2E;AAC3E,mDAAmD;AACnD,MAAM,OAAO,MAAM;IACR,QAAQ,CAA2B;IAE5C,YAAY,MAAqB,EAAE,GAAG,OAAiC;QACrE,IAAI,CAAC,QAAQ,GAAG,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,KAAK,CACd,WAAW,CAAC,IAAI,CACd,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CACzD,EACD,CAAC,CACF,CAAC;IACJ,CAAC;CACF"}
@@ -0,0 +1,9 @@
1
+ import { Ratio } from '../aggregation/ratio.ts';
2
+ import type { Penalty } from './penalty.ts';
3
+ import type { Scorer } from './scorer.ts';
4
+ export declare class Penalized<Input> implements Scorer<Input> {
5
+ #private;
6
+ constructor(scorer: Scorer<Input>, ...penalties: readonly Penalty<Input>[]);
7
+ score(input: Input): Ratio;
8
+ }
9
+ //# sourceMappingURL=penalized.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"penalized.d.ts","sourceRoot":"","sources":["../../src/scorers/penalized.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAQ1C,qBAAa,SAAS,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAIxC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,SAAS,EAAE,SAAS,OAAO,CAAC,KAAK,CAAC,EAAE;IAK1E,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAgB3B"}
@@ -0,0 +1,28 @@
1
+ import { Ratio } from "../aggregation/ratio.js";
2
+ // A scorer whose part is multiplied by the factor of each penalty, in the
3
+ // order of the penalties, for each record; the whole stays (Decorator). So a
4
+ // record that fails a gate keeps its expected values in the denominator of
5
+ // recall. A penalty on the means would give mean(a) × mean(b), so the
6
+ // penalty is per record (ADR 0004). When every factor is 1, the ratio of the
7
+ // scorer stays as it is, so an F1Ratio keeps its value (ADR 0012).
8
+ export class Penalized {
9
+ #scorer;
10
+ #penalties;
11
+ constructor(scorer, ...penalties) {
12
+ this.#scorer = scorer;
13
+ this.#penalties = penalties;
14
+ }
15
+ score(input) {
16
+ const ratio = this.#scorer.score(input);
17
+ const factors = this.#penalties.map((penalty) => {
18
+ const factor = penalty.factor(input);
19
+ if (!(factor >= 0 && factor <= 1))
20
+ throw new RangeError(`A penalty gave the factor ${factor}, which is not in [0, 1].`);
21
+ return factor;
22
+ });
23
+ if (factors.every((factor) => factor === 1))
24
+ return ratio;
25
+ return new Ratio(factors.reduce((part, factor) => part * factor, ratio.part), ratio.whole);
26
+ }
27
+ }
28
+ //# sourceMappingURL=penalized.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"penalized.js","sourceRoot":"","sources":["../../src/scorers/penalized.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAIhD,0EAA0E;AAC1E,6EAA6E;AAC7E,2EAA2E;AAC3E,sEAAsE;AACtE,6EAA6E;AAC7E,mEAAmE;AACnE,MAAM,OAAO,SAAS;IACX,OAAO,CAAgB;IACvB,UAAU,CAA4B;IAE/C,YAAY,MAAqB,EAAE,GAAG,SAAoC;QACxE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,CAAC;gBAC/B,MAAM,IAAI,UAAU,CAClB,6BAA6B,MAAM,2BAA2B,CAC/D,CAAC;YACJ,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QAC1D,OAAO,IAAI,KAAK,CACd,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,GAAG,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,EAC3D,KAAK,CAAC,KAAK,CACZ,CAAC;IACJ,CAAC;CACF"}
@@ -0,0 +1,7 @@
1
+ import { Ratio } from '../aggregation/ratio.ts';
2
+ import type { Scorer } from './scorer.ts';
3
+ export declare abstract class Penalty<Input> implements Scorer<Input> {
4
+ abstract factor(input: Input): number;
5
+ score(input: Input): Ratio;
6
+ }
7
+ //# sourceMappingURL=penalty.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"penalty.d.ts","sourceRoot":"","sources":["../../src/scorers/penalty.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAM1C,8BAAsB,OAAO,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;IAC3D,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;IAErC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAG3B"}
@@ -0,0 +1,11 @@
1
+ import { Ratio } from "../aggregation/ratio.js";
2
+ // A factor in [0, 1] for one input, as BLEU's brevity penalty is: 1 is no
3
+ // penalty. Penalized multiplies the part of a scorer by it, for each record.
4
+ // A subclass gives the factor (Template Method). As a scorer, it gives the
5
+ // factor over 1.
6
+ export class Penalty {
7
+ score(input) {
8
+ return new Ratio(this.factor(input), 1);
9
+ }
10
+ }
11
+ //# sourceMappingURL=penalty.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"penalty.js","sourceRoot":"","sources":["../../src/scorers/penalty.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGhD,0EAA0E;AAC1E,6EAA6E;AAC7E,2EAA2E;AAC3E,iBAAiB;AACjB,MAAM,OAAgB,OAAO;IAG3B,KAAK,CAAC,KAAY;QAChB,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1C,CAAC;CACF"}
@@ -0,0 +1,9 @@
1
+ import { Ratio } from '../aggregation/ratio.ts';
2
+ import type { Matches } from './matches.ts';
3
+ import type { Scorer } from './scorer.ts';
4
+ export declare class Precision<Input> implements Scorer<Input> {
5
+ #private;
6
+ constructor(matches: Matches<Input>);
7
+ score(input: Input): Ratio;
8
+ }
9
+ //# sourceMappingURL=precision.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"precision.d.ts","sourceRoot":"","sources":["../../src/scorers/precision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAG1C,qBAAa,SAAS,CAAC,KAAK,CAAE,YAAW,MAAM,CAAC,KAAK,CAAC;;gBAGxC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC;IAInC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK;CAI3B"}
@@ -0,0 +1,13 @@
1
+ import { Ratio } from "../aggregation/ratio.js";
2
+ // The hits of the stated values: Ratio(hits, stated).
3
+ export class Precision {
4
+ #matches;
5
+ constructor(matches) {
6
+ this.#matches = matches;
7
+ }
8
+ score(input) {
9
+ const { hits, stated } = this.#matches.counts(input);
10
+ return new Ratio(hits, stated);
11
+ }
12
+ }
13
+ //# sourceMappingURL=precision.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"precision.js","sourceRoot":"","sources":["../../src/scorers/precision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAIhD,sDAAsD;AACtD,MAAM,OAAO,SAAS;IACX,QAAQ,CAAiB;IAElC,YAAY,OAAuB;QACjC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAY;QAChB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrD,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACjC,CAAC;CACF"}
@@ -0,0 +1,9 @@
1
+ import { Ratio } from '../aggregation/ratio.ts';
2
+ import type { Matches } from './matches.ts';
3
+ import type { Scorer } from './scorer.ts';
4
+ export declare class Recall<Input> implements Scorer<Input> {
5
+ #private;
6
+ constructor(matches: Matches<Input>);
7
+ score(input: Input): Ratio;
8
+ }
9
+ //# sourceMappingURL=recall.d.ts.map