@zukhruf/evals 0.8.3 → 0.8.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/NOTICE +3 -3
  2. package/README.md +152 -79
  3. package/dist/aggregation/f-measure-form.d.ts +11 -0
  4. package/dist/aggregation/f-measure-form.d.ts.map +1 -0
  5. package/dist/aggregation/f-measure-form.js +25 -0
  6. package/dist/aggregation/f-measure-form.js.map +1 -0
  7. package/dist/aggregation/f-measure-ratio.d.ts +12 -0
  8. package/dist/aggregation/f-measure-ratio.d.ts.map +1 -0
  9. package/dist/aggregation/f-measure-ratio.js +21 -0
  10. package/dist/aggregation/f-measure-ratio.js.map +1 -0
  11. package/dist/aggregation/ratio-of-sums.d.ts +1 -0
  12. package/dist/aggregation/ratio-of-sums.d.ts.map +1 -1
  13. package/dist/aggregation/ratio-of-sums.js +7 -0
  14. package/dist/aggregation/ratio-of-sums.js.map +1 -1
  15. package/dist/classification/all-class-matches.d.ts +7 -0
  16. package/dist/classification/all-class-matches.d.ts.map +1 -0
  17. package/dist/classification/all-class-matches.js +14 -0
  18. package/dist/classification/all-class-matches.js.map +1 -0
  19. package/dist/classification/average.d.ts +11 -0
  20. package/dist/classification/average.d.ts.map +1 -0
  21. package/dist/classification/average.js +2 -0
  22. package/dist/classification/average.js.map +1 -0
  23. package/dist/classification/binary-average.d.ts +9 -0
  24. package/dist/classification/binary-average.d.ts.map +1 -0
  25. package/dist/classification/binary-average.js +20 -0
  26. package/dist/classification/binary-average.js.map +1 -0
  27. package/dist/classification/brier-score.d.ts +13 -0
  28. package/dist/classification/brier-score.d.ts.map +1 -0
  29. package/dist/classification/brier-score.js +52 -0
  30. package/dist/classification/brier-score.js.map +1 -0
  31. package/dist/classification/class-counts.d.ts +39 -0
  32. package/dist/classification/class-counts.d.ts.map +1 -0
  33. package/dist/classification/class-counts.js +159 -0
  34. package/dist/classification/class-counts.js.map +1 -0
  35. package/dist/classification/class-matches.d.ts +10 -0
  36. package/dist/classification/class-matches.d.ts.map +1 -0
  37. package/dist/classification/class-matches.js +21 -0
  38. package/dist/classification/class-matches.js.map +1 -0
  39. package/dist/classification/confusion-matrix.d.ts +9 -0
  40. package/dist/classification/confusion-matrix.d.ts.map +1 -0
  41. package/dist/classification/confusion-matrix.js +18 -0
  42. package/dist/classification/confusion-matrix.js.map +1 -0
  43. package/dist/classification/index.d.ts +21 -0
  44. package/dist/classification/index.d.ts.map +1 -0
  45. package/dist/classification/index.js +19 -0
  46. package/dist/classification/index.js.map +1 -0
  47. package/dist/classification/label-case.d.ts +11 -0
  48. package/dist/classification/label-case.d.ts.map +1 -0
  49. package/dist/classification/label-case.js +12 -0
  50. package/dist/classification/label-case.js.map +1 -0
  51. package/dist/classification/label-comparer.d.ts +8 -0
  52. package/dist/classification/label-comparer.d.ts.map +1 -0
  53. package/dist/classification/label-comparer.js +44 -0
  54. package/dist/classification/label-comparer.js.map +1 -0
  55. package/dist/classification/label-comparison.d.ts +12 -0
  56. package/dist/classification/label-comparison.d.ts.map +1 -0
  57. package/dist/classification/label-comparison.js +23 -0
  58. package/dist/classification/label-comparison.js.map +1 -0
  59. package/dist/classification/label-config-case.d.ts +6 -0
  60. package/dist/classification/label-config-case.d.ts.map +1 -0
  61. package/dist/classification/label-config-case.js +13 -0
  62. package/dist/classification/label-config-case.js.map +1 -0
  63. package/dist/classification/label-vars-case.d.ts +6 -0
  64. package/dist/classification/label-vars-case.d.ts.map +1 -0
  65. package/dist/classification/label-vars-case.js +17 -0
  66. package/dist/classification/label-vars-case.js.map +1 -0
  67. package/dist/classification/label.d.ts +4 -0
  68. package/dist/classification/label.d.ts.map +1 -0
  69. package/dist/classification/label.js +16 -0
  70. package/dist/classification/label.js.map +1 -0
  71. package/dist/classification/labels-match.d.ts +7 -0
  72. package/dist/classification/labels-match.d.ts.map +1 -0
  73. package/dist/classification/labels-match.js +13 -0
  74. package/dist/classification/labels-match.js.map +1 -0
  75. package/dist/classification/macro-average.d.ts +6 -0
  76. package/dist/classification/macro-average.d.ts.map +1 -0
  77. package/dist/classification/macro-average.js +15 -0
  78. package/dist/classification/macro-average.js.map +1 -0
  79. package/dist/classification/micro-average.d.ts +6 -0
  80. package/dist/classification/micro-average.d.ts.map +1 -0
  81. package/dist/classification/micro-average.js +16 -0
  82. package/dist/classification/micro-average.js.map +1 -0
  83. package/dist/classification/per-class.d.ts +12 -0
  84. package/dist/classification/per-class.d.ts.map +1 -0
  85. package/dist/classification/per-class.js +17 -0
  86. package/dist/classification/per-class.js.map +1 -0
  87. package/dist/classification/probability-comparer.d.ts +8 -0
  88. package/dist/classification/probability-comparer.d.ts.map +1 -0
  89. package/dist/classification/probability-comparer.js +35 -0
  90. package/dist/classification/probability-comparer.js.map +1 -0
  91. package/dist/classification/probability-comparison.d.ts +10 -0
  92. package/dist/classification/probability-comparison.d.ts.map +1 -0
  93. package/dist/classification/probability-comparison.js +11 -0
  94. package/dist/classification/probability-comparison.js.map +1 -0
  95. package/dist/classification/read-labels.d.ts +3 -0
  96. package/dist/classification/read-labels.d.ts.map +1 -0
  97. package/dist/classification/read-labels.js +11 -0
  98. package/dist/classification/read-labels.js.map +1 -0
  99. package/dist/classification/weighted-average.d.ts +6 -0
  100. package/dist/classification/weighted-average.d.ts.map +1 -0
  101. package/dist/classification/weighted-average.js +20 -0
  102. package/dist/classification/weighted-average.js.map +1 -0
  103. package/dist/index.d.ts +5 -1
  104. package/dist/index.d.ts.map +1 -1
  105. package/dist/index.js +4 -1
  106. package/dist/index.js.map +1 -1
  107. package/dist/promptfoo/index.d.ts +1 -0
  108. package/dist/promptfoo/index.d.ts.map +1 -1
  109. package/dist/promptfoo/index.js +1 -0
  110. package/dist/promptfoo/index.js.map +1 -1
  111. package/dist/promptfoo/scorer-assertion.d.ts +18 -0
  112. package/dist/promptfoo/scorer-assertion.d.ts.map +1 -0
  113. package/dist/{structured/promptfoo → promptfoo}/scorer-assertion.js +5 -11
  114. package/dist/promptfoo/scorer-assertion.js.map +1 -0
  115. package/dist/scorers/f-beta.d.ts +13 -0
  116. package/dist/scorers/f-beta.d.ts.map +1 -0
  117. package/dist/scorers/f-beta.js +22 -0
  118. package/dist/scorers/f-beta.js.map +1 -0
  119. package/dist/scorers/f1.d.ts +6 -6
  120. package/dist/scorers/f1.d.ts.map +1 -1
  121. package/dist/scorers/f1.js +5 -9
  122. package/dist/scorers/f1.js.map +1 -1
  123. package/dist/scorers/penalized.d.ts.map +1 -1
  124. package/dist/scorers/penalized.js +2 -1
  125. package/dist/scorers/penalized.js.map +1 -1
  126. package/dist/scoring/bootstrap.js +3 -5
  127. package/dist/scoring/bootstrap.js.map +1 -1
  128. package/dist/scoring/comparer.d.ts +5 -0
  129. package/dist/scoring/comparer.d.ts.map +1 -0
  130. package/dist/scoring/comparer.js +2 -0
  131. package/dist/scoring/comparer.js.map +1 -0
  132. package/dist/scoring/grouped.d.ts +9 -0
  133. package/dist/scoring/grouped.d.ts.map +1 -0
  134. package/dist/scoring/grouped.js +20 -0
  135. package/dist/scoring/grouped.js.map +1 -0
  136. package/dist/scoring/scorer-set.d.ts +13 -0
  137. package/dist/scoring/scorer-set.d.ts.map +1 -0
  138. package/dist/scoring/scorer-set.js +55 -0
  139. package/dist/scoring/scorer-set.js.map +1 -0
  140. package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts +4 -1
  141. package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts.map +1 -1
  142. package/dist/structured/comparison/comparators/token-overlap-at-least.js +7 -4
  143. package/dist/structured/comparison/comparators/token-overlap-at-least.js.map +1 -1
  144. package/dist/structured/comparison/json-value.d.ts +1 -0
  145. package/dist/structured/comparison/json-value.d.ts.map +1 -1
  146. package/dist/structured/comparison/json-value.js +11 -2
  147. package/dist/structured/comparison/json-value.js.map +1 -1
  148. package/dist/structured/comparison/tree-walk.d.ts +5 -5
  149. package/dist/structured/comparison/tree-walk.d.ts.map +1 -1
  150. package/dist/structured/comparison/tree-walk.js +8 -4
  151. package/dist/structured/comparison/tree-walk.js.map +1 -1
  152. package/dist/structured/cpython/python-text.d.ts +0 -1
  153. package/dist/structured/cpython/python-text.d.ts.map +1 -1
  154. package/dist/structured/cpython/python-text.js +1 -18
  155. package/dist/structured/cpython/python-text.js.map +1 -1
  156. package/dist/structured/index.d.ts +0 -2
  157. package/dist/structured/index.d.ts.map +1 -1
  158. package/dist/structured/index.js +0 -2
  159. package/dist/structured/index.js.map +1 -1
  160. package/dist/structured/scorers/token-f1.d.ts +3 -1
  161. package/dist/structured/scorers/token-f1.d.ts.map +1 -1
  162. package/dist/structured/scorers/token-f1.js +8 -5
  163. package/dist/structured/scorers/token-f1.js.map +1 -1
  164. package/dist/structured/scoring/per-field.d.ts +3 -6
  165. package/dist/structured/scoring/per-field.d.ts.map +1 -1
  166. package/dist/structured/scoring/per-field.js +8 -17
  167. package/dist/structured/scoring/per-field.js.map +1 -1
  168. package/package.json +18 -5
  169. package/dist/aggregation/f1-ratio.d.ts +0 -8
  170. package/dist/aggregation/f1-ratio.d.ts.map +0 -1
  171. package/dist/aggregation/f1-ratio.js +0 -24
  172. package/dist/aggregation/f1-ratio.js.map +0 -1
  173. package/dist/structured/promptfoo/scorer-assertion.d.ts +0 -22
  174. package/dist/structured/promptfoo/scorer-assertion.d.ts.map +0 -1
  175. package/dist/structured/promptfoo/scorer-assertion.js.map +0 -1
  176. package/dist/structured/scoring/scorer-set.d.ts +0 -15
  177. package/dist/structured/scoring/scorer-set.d.ts.map +0 -1
  178. package/dist/structured/scoring/scorer-set.js +0 -60
  179. package/dist/structured/scoring/scorer-set.js.map +0 -1
package/NOTICE CHANGED
@@ -1,4 +1,4 @@
1
- @zukhruf/structured-eval contains a TypeScript port of the scorer of the
1
+ @zukhruf/evals contains a TypeScript port of the scorer of the
2
2
  Structured Output Benchmark (SOB): evaluate.py at commit da785a8 of
3
3
  https://github.com/interfazeai/sob. That code has this licence:
4
4
 
@@ -24,7 +24,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24
24
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25
25
  SOFTWARE.
26
26
 
27
- @zukhruf/structured-eval contains test cases from the JSON-Schema-Test-Suite:
27
+ @zukhruf/evals contains test cases from the JSON-Schema-Test-Suite:
28
28
  tests/draft2020-12/optional/ecmascript-regex.json and
29
29
  tests/draft7/optional/ecmascript-regex.json at commit
30
30
  245a45abc90190c0bff04d97e6d2b541b4bb9f12 of
@@ -53,7 +53,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
53
53
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
54
54
  THE SOFTWARE.
55
55
 
56
- @zukhruf/structured-eval contains the JSON Schema meta-schemas of draft 7 and
56
+ @zukhruf/evals contains the JSON Schema meta-schemas of draft 7 and
57
57
  draft 2020-12 (src/schema/draft-07-schema.ts and
58
58
  src/schema/draft-2020-12-schema.ts), copied from jsonschema-specifications
59
59
  2025.9.1 (https://github.com/python-jsonschema/jsonschema-specifications). That
package/README.md CHANGED
@@ -6,18 +6,18 @@ The answers of evals have different shapes: a structured record, a label, a rank
6
6
 
7
7
  ## The parts
8
8
 
9
- | Import path | Job |
10
- | -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
11
- | `@zukhruf/evals` | The core: scorers, ratios, gates, scorer sets, scorecards, totals and bootstrap intervals. |
12
- | `@zukhruf/evals/promptfoo` | The adapter that makes a scorer or a scorer set a promptfoo assertion. |
13
- | `@zukhruf/evals/structured` | Compares a structured value with the value that a person labelled. |
14
- | `@zukhruf/evals/classification` (next) | Compares the stated labels of an answer with its expected labels, as scikit-learn's `sklearn.metrics` does. Designed in ADR 0003; built in backlog #3889. |
15
- | `@zukhruf/evals/ranking` (later) | Compares a ranked list with judged items, as trec_eval does. |
16
- | `@zukhruf/evals/clustering` (later) | Compares a partition with a labelled partition, as the CoNLL reference scorer does. |
9
+ | Import path | Job |
10
+ | ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------- |
11
+ | `@zukhruf/evals` | The core: scorers, ratios, gates, scorer sets, scorecards, totals and bootstrap intervals. |
12
+ | `@zukhruf/evals/promptfoo` | The adapter that makes a scorer or a scorer set a promptfoo assertion. |
13
+ | `@zukhruf/evals/structured` | Compares a structured value with the value that a person labelled. |
14
+ | `@zukhruf/evals/classification` | Compares the stated labels of an answer with its expected labels, as scikit-learn's `sklearn.metrics` does (ADR 0003). |
15
+ | `@zukhruf/evals/ranking` (later) | Compares a ranked list with judged items, as trec_eval does. |
16
+ | `@zukhruf/evals/clustering` (later) | Compares a partition with a labelled partition, as the CoNLL reference scorer does. |
17
17
 
18
18
  A subpath imports only the core. It never imports another subpath.
19
19
 
20
- The first release (backlog #3888) has one family, so the core holds only the parts that are generic today: `Ratio`, `Counts`, `RatioOfSums`, `Scorecard`, `ScorecardScorer`, `CaseReader`, `Totals` and `Bootstrap`. The scorers, the penalties, `ScorerSet` and `ScorerAssertion` read a structured `Comparison`, so they are in `./structured` until `./classification` (#3889) brings the second input. Then they move into the core with a type parameter, as the API below shows (ADR 0002).
20
+ The core holds the parts that all families share: the ratios, the scorers, the penalties, `Comparer`, `Grouped`, `ScorerSet`, `Totals` and `Bootstrap`. A part that reads an input has a type parameter for that input. So a scorer of one family cannot read the input of another family (ADR 0002). `./promptfoo` holds `ScorerAssertion`. The families are `./structured` and `./classification`.
21
21
 
22
22
  ## Score one answer
23
23
 
@@ -41,6 +41,11 @@ new Precision(new ClassMatches('home')).score(comparison); // Ratio { part: 0, w
41
41
 
42
42
  A `Ratio` is a part and a whole. Its value is the part divided by the whole, and 0 when the whole is 0. `Precision`, `Recall`, `F1` and `FBeta` read the counts of a `Matches`: the hits, the stated and the expected. Each family gives its own matches, and these scorers take any of them.
43
43
 
44
+ The ratio of `new FBeta(matches, { beta, form })` has the part (1 + β²) × hits and the whole β² × expected + stated, so many of them add up to the micro F-measure. `F1` is `FBeta` with β = 1. The form gives the value (ADR 0003):
45
+
46
+ - `new CountsForm()` is the default. It is scikit-learn's arithmetic, and its value is the part divided by the whole: 0 when the whole is 0. So 1 hit of 5 stated and 1 expected gives 1/3.
47
+ - `new PrecisionRecallForm()` is (1 + β²)·P·R / (β²·P + R): 0 when there are no hits. SOB's evaluate.py uses it. The same counts give 0.33333333333333337.
48
+
44
49
  ## Combine scorers
45
50
 
46
51
  A penalty is a factor in [0, 1] for one input, as BLEU's brevity penalty is. The factor 1 is no penalty. A penalty is also a scorer: its ratio is `Ratio(factor, 1)`. A gate is the penalty whose factor is 0 or 1. A gate says what it checks (`holds`) and why it fails (`explain`).
@@ -53,9 +58,9 @@ A penalty is a factor in [0, 1] for one input, as BLEU's brevity penalty is. The
53
58
 
54
59
  ## Score with many scorers at once
55
60
 
56
- A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. The family's comparer makes the input once, and each scorer reads it.
61
+ A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. Each gate gives a check, whose `name` is the label of the gate. The `comparer` of the family is required, because the core cannot know a family: `new ScorerSet(scorers, { comparer: new LabelComparer() })`. It makes the input once, and each scorer reads it.
57
62
 
58
- A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core, so a `ScorerSet` holds them beside plain scorers (Inspect AI's `grouped()` applies a metric to each group in the same way).
63
+ A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core: a subclass gives its groups with `groups(input)`, and `ratios` drops a group whose whole is 0. So a `ScorerSet` holds them beside plain scorers (Inspect AI's `grouped()` applies a metric to each group in the same way).
59
64
 
60
65
  ```ts
61
66
  import { Recall, ScorerSet } from '@zukhruf/evals';
@@ -81,6 +86,10 @@ const pageKinds = new ScorerSet(
81
86
  - A plain scorer gives its counts, such as Ratio(2, 3). `Totals` then gives the micro average.
82
87
  - A `SampleAveraged` scorer gives its value over 1. `Totals` then gives the samples average: the mean of the values of the answers.
83
88
 
89
+ `Totals` adds with CPython's compensated `sum()`. A sum of integer weights is exact, so its means and accuracies equal scikit-learn's for integer weights. A sum of float weights can differ from numpy's pairwise sum in the last bit. For float weights, use `ClassCounts` and `BrierScore.mean` of `./classification`.
90
+
91
+ `RatioOfSums.of(ratios)` adds a list of ratios in one call.
92
+
84
93
  `Bootstrap` gives the interval of one name over the scorecards, with `@zukhruf/statistics`. You give it the number of resamples, the random index and the method of the interval.
85
94
 
86
95
  ## Compare structured values
@@ -91,67 +100,92 @@ It needs two optional peers: `npm install @zukhruf/evals @cfworker/json-schema m
91
100
 
92
101
  ## Score labels
93
102
 
94
- > **Designed, not in this release.** `./classification` is built in backlog #3889 (ADR 0003). This section is its API on paper.
103
+ `./classification` compares the stated labels of an answer with its expected labels. One label is a set of one label, so one model serves multiclass answers (one class) and multilabel answers (a set of classes). The arithmetic follows scikit-learn 1.9.1, bit for bit (ADR 0003). The golden tests compare each number with the output of `src/classification/fixtures/sklearn/generate.py`.
95
104
 
96
- `./classification` compares the stated labels of an answer with its expected labels. One label is a set of one label, so one model serves a class (multiclass) and a set of classes (multilabel). The arithmetic follows scikit-learn 1.9.1, bit for bit.
105
+ It needs one optional peer: `npm install @zukhruf/evals @stdlib/blas-ext-base-gsumpw`. That package gives numpy's pairwise sum.
97
106
 
98
107
  ### For each answer
99
108
 
100
- `LabelComparer` reads the stated labels from the answer and gives a `LabelComparison`. Its matches count one answer:
109
+ A case holds the expected labels and the weight: `new LabelCase({ expected, weight })`. `expected` is a label or a list of labels. `LabelVarsCase` reads `vars.expected` and `vars.weight` of a test (a test without a weight weighs 1). `LabelConfigCase` reads `config.expected` of an assertion, with the weight 1.
110
+
111
+ `LabelComparer` reads the stated labels of an answer and gives a `LabelComparison`. It reads a label or a list of labels as they are. It reads text in this order:
112
+
113
+ 1. A JSON array of labels gives those labels.
114
+ 2. A number gives a number label, but only when each expected label of the case is a number. So `"7"` matches the label `7` and the label `"7"`.
115
+ 3. Other text is one string label, without its white space at the start and the end.
116
+
117
+ `comparison.only(label)` gives the comparison of one class: each side keeps the label when it has it.
118
+
119
+ The matches of a comparison count one answer:
101
120
 
102
121
  | Matches | Hits | Stated | Expected |
103
122
  | ------------------------- | -------------------------------- | ------------------------ | ---------------------- |
104
123
  | `new ClassMatches(label)` | 1 when both sides have the label | 1 when the answer has it | 1 when the case has it |
105
124
  | `new AllClassMatches()` | the labels that both sides have | the stated labels | the expected labels |
106
125
 
107
- `LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `Totals` of its ratios gives the accuracy (for a set of classes: the subset accuracy of scikit-learn). `new PerClass(scorer, { labels })` gives one ratio for each class, under the label `${class} ${label}`.
126
+ `LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `new PerClass(scorer, { labels })` gives one ratio for each class, under the label `${class} ${label}`. Without `labels`, the classes are the labels of both sides of the answer, in the order of numpy's `unique()` (numbers by value, strings by code point).
127
+
128
+ `Totals` of these ratios gives scikit-learn's numbers, bit for bit, for integer weights:
108
129
 
109
- So a promptfoo run gives the micro precision, recall and F1, and the precision and recall of each class, in its column headers.
130
+ - `ClassMatches` gives the precision, recall and F1 of each class (`average=None`).
131
+ - `AllClassMatches` gives the micro average.
132
+ - `LabelsMatch` gives `accuracy_score` (for multilabel answers, the subset accuracy).
133
+
134
+ So a promptfoo run gives the micro precision, recall and F1, and the score of each class, in its column headers.
110
135
 
111
136
  ### For a whole run
112
137
 
113
- A macro or weighted average is not a ratio of sums: it takes the value of each class first. `ClassCounts` keeps the counts of each class (true positives, predicted, true), as scikit-learn's `multilabel_confusion_matrix` gives them. Add each comparison, then read the metrics:
138
+ A macro average or a weighted average is not a ratio of sums: it needs the score of each class first. `ClassCounts` keeps the counts of each class: the true positives (hits), the predicted (stated) and the true (expected). It adds the weight of each answer in the order of the answers, as numpy's `bincount` does. torchmetrics does the same with `update()` and `compute()`.
114
139
 
115
140
  ```ts
116
141
  import {
117
142
  ClassCounts,
118
- CountsForm,
143
+ LabelCase,
144
+ LabelComparer,
119
145
  MacroAverage,
120
146
  WeightedAverage,
121
147
  } from '@zukhruf/evals/classification';
122
148
 
123
- const counts = new ClassCounts();
124
- for (const comparison of comparisons) counts.add(comparison);
149
+ const counts = ClassCounts.of({
150
+ expected: ['a', 'b', 'b'],
151
+ stated: ['a', 'a', 'b'],
152
+ });
125
153
 
126
154
  counts.accuracy(); // accuracy_score
127
155
  counts.fBeta({ beta: 1, average: new MacroAverage() }); // f1_score(average='macro')
128
156
  counts.precision({ average: new WeightedAverage() }); // precision_score(average='weighted')
129
- counts.fBeta({
130
- beta: 0.5,
131
- average: new MacroAverage(),
132
- form: new CountsForm(),
133
- });
134
- counts.confusionMatrix(); // confusion_matrix, labels in sorted order
157
+ counts.confusionMatrix().rows(); // confusion_matrix, labels in sorted order
158
+
159
+ // The same counts, one answer at a time:
160
+ const run = new ClassCounts({ multilabel: false });
161
+ const testCase = new LabelCase({ expected: 'a', weight: 1 });
162
+ run.add(new LabelComparer().compare('a', testCase), testCase.weight);
135
163
  ```
136
164
 
137
- - `average` is a Strategy: `new MicroAverage()`, `new MacroAverage()`, `new WeightedAverage()` or `new BinaryAverage(positiveLabel)`. The samples average is a `SampleAveraged` scorer, because it needs the value of each answer.
138
- - `zeroDivision` is the value of a class score whose denominator is 0: 0 (the default, as scikit-learn's `"warn"` gives), 1 or `NaN`.
139
- - `form` is the arithmetic of the F-measure (ADR 0003). `new CountsForm()` is scikit-learn's and the default; it also gives the bits of TREC's F0.5 for integer counts. `new PrecisionRecallForm()` is van Rijsbergen's and SOB's.
140
- - `ClassCounts.of(stated, expected)` builds the counts from two arrays of labels, as a batch.
165
+ - `ClassCounts.of({ expected, stated, weights })` builds multiclass counts from two arrays of labels. `weights` is optional, and an answer without a weight weighs 1.
166
+ - `ClassCounts.ofIndicators({ expected, stated, weights })` builds multilabel counts from two label indicator matrices. Each column is a class, also when no answer has it.
167
+ - `new ClassCounts({ multilabel })` starts empty, and `add(comparison, weight)` adds one answer. Multiclass counts refuse an answer with no label or more than one label on a side.
168
+ - `average` is a Strategy: `new MicroAverage()`, `new MacroAverage()`, `new WeightedAverage()` or `new BinaryAverage(positiveLabel)`. `Average.of(classes, score)` gives one number from the score of each class. The micro average adds the counts with numpy's pairwise sum. The macro average is numpy's `nanmean`. The weighted average is scikit-learn's `_nanaverage`: it drops each NaN score with its weight, and it gives the plain mean when the weights add up to 0. `BinaryAverage` throws when scikit-learn raises: for multilabel counts, for more than two classes, and for a positive label that is not one of two classes. The samples average is a `SampleAveraged` scorer, because it needs the value of each answer.
169
+ - `labels` selects the classes to score, in its order. By default it is every class, in sorted order.
170
+ - `zeroDivision` is the score of a class whose denominator is 0: 0 (the default, as scikit-learn's `"warn"` gives), 1 or `NaN`. A macro average and a weighted average skip a `NaN` score.
171
+ - `form` is the arithmetic of the F-measure (ADR 0003). `new CountsForm()` is scikit-learn's and the default. It also gives the bits of TREC's F0.5 for integer counts. `new PrecisionRecallForm()` is van Rijsbergen's and SOB's.
172
+ - `confusionMatrix()` throws for multilabel counts, as scikit-learn raises.
141
173
 
142
174
  ### Probabilities
143
175
 
144
- `BrierScore` compares the stated probability of each class with the expected labels, as scikit-learn's `brier_score_loss` does. Its ratio is the score of one answer over 1, so `Totals` gives the mean.
176
+ `ProbabilityComparer` reads the stated probability of each class of an answer and gives a `ProbabilityComparison`. The answer is a `Map`, a plain object or the JSON text of an object. A key is a number label when the expected label is a number. A probability outside [0, 1] throws a `RangeError`. The case has one expected label.
177
+
178
+ `BrierScore` follows scikit-learn's `brier_score_loss`. For one answer, it adds the squares of (1 for the expected label, else 0, minus the probability) over the classes in sorted order, with numpy's pairwise sum. For two classes, it multiplies the sum by 0.5. `new BrierScore({ positiveLabel })` reads only the probability p of the positive label, and the two classes are 1 − p and p. Its ratio is the score of one answer over 1. `brierScore.mean(comparisons, weights)` gives the score of a whole run with numpy's pairwise sum, bit for bit.
145
179
 
146
180
  ## Use it with promptfoo
147
181
 
148
- > In this release, `ScorerAssertion` is exported by `@zukhruf/evals/structured`, and it scores a structured value (ADR 0002). The sample below shows the API after #3889, where it moves to `./promptfoo` and also scores labels. [docs/structured.md](docs/structured.md) has the structured sample of today.
182
+ `ScorerAssertion` is in `./promptfoo`. It takes the comparer and the case reader of a family. The sample below scores labels. [docs/structured.md](docs/structured.md) has a structured sample.
149
183
 
150
184
  `PromptfooAssertion` adapts a scorer set to a per-row assertion. Each ratio becomes a named score with its whole as the weight, so the column header shows the same ratio of sums as `Totals`. It reads the case from the config of the assertion.
151
185
 
152
186
  `ScorerAssertion` holds one scorer with its `weight` (how much it counts in the score of the test) and its `threshold` (when it passes). Both numbers are required. It reads the case from the vars of the test.
153
187
 
154
- promptfoo stores the config of a run as JSON, and `--resume` and `--retry-errors` build the run from that stored config. A function does not survive JSON, so a config holds no scorer assertion. An `AssertionModule` holds the scorer assertions by name, in a module that promptfoo loads. Its `assert(...names)` gives JSON assertions whose `value` is the file reference `file://<module>:<name>`. promptfoo loads the function from the default export of the module when it grades, on the first run and on a resumed run (structured-eval ADR 0013).
188
+ promptfoo stores the config of a run as JSON, and `--resume` and `--retry-errors` build the run from that stored config. A function does not survive JSON, so a config holds no scorer assertion. An `AssertionModule` holds the scorer assertions by name, in a module that promptfoo loads. Its `assert(...names)` gives JSON assertions whose `value` is the file reference `file://<module>:<name>`. promptfoo loads the function from the default export of the module when it grades, on the first run and on a resumed run (structured ADR 0013 in docs/adr/structured).
155
189
 
156
190
  ```ts
157
191
  // page-kinds/scorers.ts: the module that promptfoo loads
@@ -313,6 +347,7 @@ export interface Comparer<Case extends Weighted, Input> {
313
347
  }
314
348
 
315
349
  export type Check = {
350
+ readonly name: string;
316
351
  readonly passed: boolean;
317
352
  readonly score: number;
318
353
  readonly reason: string;
@@ -330,7 +365,11 @@ export interface ScorecardScorer<Case extends Weighted> {
330
365
  }
331
366
  /** Gives one ratio for each group of an input, such as each field or each class. */
332
367
  export abstract class Grouped<Input> {
333
- abstract ratios(input: Input, label: string): [string, Ratio][];
368
+ constructor(scorer: Scorer<Input>);
369
+ /** The groups of an input (Template Method). */
370
+ protected abstract groups(input: Input): Iterable<[string, Input]>;
371
+ /** The ratio of the scorer on each group, under `${group} ${label}`. A group whose whole is 0 gives no ratio. */
372
+ ratios(input: Input, label: string): [string, Ratio][];
334
373
  }
335
374
  export class ScorerSet<
336
375
  Case extends Weighted,
@@ -344,6 +383,8 @@ export class ScorerSet<
344
383
 
345
384
  /** Adds ratios: the sum of the parts divided by the sum of the wholes. */
346
385
  export class RatioOfSums {
386
+ /** The ratio of sums of a list of ratios, in one call. */
387
+ static of(ratios: Iterable<Ratio>): RatioOfSums;
347
388
  add(ratio: Ratio): void;
348
389
  get ratio(): Ratio;
349
390
  get value(): number;
@@ -404,43 +445,55 @@ export type FileAssertion = {
404
445
  readonly threshold: number;
405
446
  readonly value: string; // file://<module>:<name>
406
447
  };
407
- /** The module that promptfoo loads the function of each scorer assertion from (structured-eval ADR 0013). */
448
+ /** The vars of the test that promptfoo gives a javascript assertion of a shared defaultTest. */
449
+ export type VarsContext = {
450
+ readonly vars: Readonly<Record<string, unknown>>;
451
+ };
452
+ /** What an assertion module needs of an assertion. A ScorerAssertion is one. */
453
+ export type ModuleAssertion = {
454
+ readonly metric?: string | undefined;
455
+ readonly weight: number;
456
+ readonly threshold: number;
457
+ readonly value: (output: string, context: VarsContext) => number;
458
+ };
459
+ /** The module that promptfoo loads the function of each scorer assertion from (structured ADR 0013 in docs/adr/structured). */
408
460
  export class AssertionModule<Name extends string> {
409
- constructor(
410
- url: string,
411
- assertions: Readonly<Record<Name, ScorerAssertion<Weighted, unknown>>>,
412
- );
461
+ constructor(url: string, assertions: Readonly<Record<Name, ModuleAssertion>>);
413
462
  /** The default export of the module: the value function of each name. */
414
- readonly functions: Readonly<
415
- Record<string, ScorerAssertion<Weighted, unknown>['value']>
416
- >;
463
+ readonly functions: Readonly<Record<string, ModuleAssertion['value']>>;
417
464
  assert(...names: Name[]): FileAssertion[];
418
465
  }
419
466
 
420
- // @zukhruf/evals/classification (designed; built in #3889)
467
+ // @zukhruf/evals/classification
421
468
 
422
- /** A label: an int or a string, as scikit-learn reads a label. */
469
+ /** A label: a number or a string, as scikit-learn reads a label. */
423
470
  export type Label = number | string;
424
471
 
425
472
  /** The expected labels of one test, and its weight. */
426
473
  export class LabelCase implements Weighted {
427
- constructor(expected: Label | readonly Label[], weight?: number);
474
+ constructor(options: { expected: Label | readonly Label[]; weight: number });
428
475
  readonly expected: ReadonlySet<Label>;
429
476
  readonly weight: number;
430
477
  }
431
- /** Reads `vars.expected` (a label or a list of labels) and `vars.weight`. */
478
+ /** Reads `vars.expected` (a label or a list of labels) and `vars.weight` (1 when none). */
432
479
  export class LabelVarsCase implements CaseReader<LabelCase> {}
433
- /** Reads `config.expected`. */
480
+ /** Reads `config.expected`, with the weight 1. */
434
481
  export class LabelConfigCase implements CaseReader<LabelCase> {}
435
482
 
436
483
  export class LabelComparison {
484
+ constructor(sides: {
485
+ stated: ReadonlySet<Label>;
486
+ expected: ReadonlySet<Label>;
487
+ });
437
488
  readonly stated: ReadonlySet<Label>;
438
489
  readonly expected: ReadonlySet<Label>;
490
+ /** The labels of both sides, in the order of numpy's unique(). */
491
+ labels(): Label[];
492
+ /** The comparison of one class. */
493
+ only(label: Label): LabelComparison;
439
494
  }
440
- /** Reads the stated labels of an answer: a label, a list of labels, or their JSON text. */
441
- export class LabelComparer implements Comparer<LabelCase, LabelComparison> {
442
- constructor(options?: { normalize?: (label: string) => string });
443
- }
495
+ /** Reads the stated labels: a label, a list of labels, a JSON array of labels, a number, or one string label. */
496
+ export class LabelComparer implements Comparer<LabelCase, LabelComparison> {}
444
497
 
445
498
  export class ClassMatches implements Matches<LabelComparison> {
446
499
  constructor(label: Label);
@@ -451,12 +504,20 @@ export class LabelsMatch extends Gate<LabelComparison> {}
451
504
  export class PerClass extends Grouped<LabelComparison> {
452
505
  constructor(
453
506
  scorer: Scorer<LabelComparison>,
454
- options: { labels: readonly Label[] },
507
+ options?: { labels?: readonly Label[] },
455
508
  );
456
509
  }
457
510
 
511
+ /** The classes that an average reads. */
512
+ export type Classes = {
513
+ readonly selected: ReadonlyMap<Label, Counts>;
514
+ readonly present: ReadonlyMap<Label, Counts>;
515
+ readonly multilabel: boolean;
516
+ };
458
517
  /** How the scores of the classes become one number (Strategy). */
459
- export interface Average {}
518
+ export interface Average {
519
+ of(classes: Classes, score: (counts: Counts) => number): number;
520
+ }
460
521
  export class MicroAverage implements Average {}
461
522
  export class MacroAverage implements Average {}
462
523
  export class WeightedAverage implements Average {}
@@ -464,33 +525,33 @@ export class BinaryAverage implements Average {
464
525
  constructor(positiveLabel: Label);
465
526
  }
466
527
 
528
+ type ScoreOptions = {
529
+ average: Average;
530
+ labels?: readonly Label[];
531
+ /** 0 (the default), 1 or NaN. */
532
+ zeroDivision?: number;
533
+ };
467
534
  /** The counts of each class over many answers: true positives, predicted and true. */
468
535
  export class ClassCounts {
469
- static of(
470
- stated: readonly (Label | readonly Label[])[],
471
- expected: readonly (Label | readonly Label[])[],
472
- ): ClassCounts;
473
- add(comparison: LabelComparison): void;
536
+ constructor(options: { multilabel: boolean });
537
+ static of(answers: {
538
+ expected: readonly Label[];
539
+ stated: readonly Label[];
540
+ weights?: readonly number[];
541
+ }): ClassCounts;
542
+ static ofIndicators(answers: {
543
+ expected: readonly (readonly number[])[];
544
+ stated: readonly (readonly number[])[];
545
+ weights?: readonly number[];
546
+ }): ClassCounts;
547
+ add(comparison: LabelComparison, weight: number): void;
474
548
  labels(): Label[];
475
549
  accuracy(): number;
476
- precision(options: {
477
- average: Average;
478
- zeroDivision?: number;
479
- labels?: readonly Label[];
480
- }): number;
481
- recall(options: {
482
- average: Average;
483
- zeroDivision?: number;
484
- labels?: readonly Label[];
485
- }): number;
486
- fBeta(options: {
487
- beta: number;
488
- average: Average;
489
- zeroDivision?: number;
490
- labels?: readonly Label[];
491
- form?: FMeasureForm;
492
- }): number;
550
+ precision(options: ScoreOptions): number;
551
+ recall(options: ScoreOptions): number;
552
+ fBeta(options: ScoreOptions & { beta: number; form?: FMeasureForm }): number;
493
553
  support(): ReadonlyMap<Label, number>;
554
+ /** Throws for multilabel counts. */
494
555
  confusionMatrix(): ConfusionMatrix;
495
556
  }
496
557
  export class ConfusionMatrix {
@@ -499,12 +560,24 @@ export class ConfusionMatrix {
499
560
  rows(): number[][];
500
561
  }
501
562
 
502
- /** The stated probability of each class of one answer. */
563
+ /** The stated probability of each class of one answer, and its expected label. */
503
564
  export class ProbabilityComparison {
504
565
  readonly probabilities: ReadonlyMap<Label, number>;
505
- readonly expected: ReadonlySet<Label>;
566
+ readonly expected: Label;
567
+ }
568
+ /** Reads the probability of each label from a Map, an object or its JSON text. */
569
+ export class ProbabilityComparer implements Comparer<
570
+ LabelCase,
571
+ ProbabilityComparison
572
+ > {}
573
+ export class BrierScore implements Scorer<ProbabilityComparison> {
574
+ constructor(options?: { positiveLabel?: Label });
575
+ /** The score of a whole run, with numpy's pairwise sum. */
576
+ mean(
577
+ comparisons: readonly ProbabilityComparison[],
578
+ weights?: readonly number[],
579
+ ): number;
506
580
  }
507
- export class BrierScore implements Scorer<ProbabilityComparison> {}
508
581
  ```
509
582
 
510
583
  ## Who uses each part
@@ -520,7 +593,7 @@ The consumers below come from the search of 2026-11-06 (eight evals in four repo
520
593
  | relevance-filtering eval | `./classification` (`BinaryAverage`, F0.5 with `CountsForm`, `BrierScore`) | mutex #3891 |
521
594
  | event-detection eval | nothing yet: its MUC, B³, CEAF-e and BLANC wait for `./clustering` | — |
522
595
 
523
- The build is in two items: mutex #3888 (the core, `./promptfoo` and the move of `./structured`) and mutex #3889 (`./classification`).
596
+ The package was built in two items: mutex #3888 (the core, `./promptfoo` and the move of `./structured`) and mutex #3889 (the generic core, the F-measure form and `./classification`).
524
597
 
525
598
  ## Documentation
526
599
 
@@ -0,0 +1,11 @@
1
+ import type { Counts } from './counts.ts';
2
+ export interface FMeasureForm {
3
+ of(counts: Counts, beta: number): number;
4
+ }
5
+ export declare class CountsForm implements FMeasureForm {
6
+ of({ hits, stated, expected }: Counts, beta: number): number;
7
+ }
8
+ export declare class PrecisionRecallForm implements FMeasureForm {
9
+ of({ hits, stated, expected }: Counts, beta: number): number;
10
+ }
11
+ //# sourceMappingURL=f-measure-form.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"f-measure-form.d.ts","sourceRoot":"","sources":["../../src/aggregation/f-measure-form.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAK1C,MAAM,WAAW,YAAY;IAC3B,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CAC1C;AAMD,qBAAa,UAAW,YAAW,YAAY;IAC7C,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;CAI7D;AAKD,qBAAa,mBAAoB,YAAW,YAAY;IACtD,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;CAO7D"}
@@ -0,0 +1,25 @@
1
+ import { Ratio } from "./ratio.js";
2
+ // (1 + β²)·hits / (β²·expected + stated), in the order of the operations of
3
+ // scikit-learn 1.9.1's fbeta_score (sklearn/metrics/_classification.py). It
4
+ // is the part of an F-measure ratio divided by its whole, and 0 when the
5
+ // whole is 0.
6
+ export class CountsForm {
7
+ of({ hits, stated, expected }, beta) {
8
+ const beta2 = beta ** 2;
9
+ return new Ratio((1 + beta2) * hits, beta2 * expected + stated).value;
10
+ }
11
+ }
12
+ // (1 + β²)·P·R / (β²·P + R), precision and recall first (van Rijsbergen
13
+ // 1979), and 0 when there are no hits. With β = 1 it is 2·p·r/(p + r), the
14
+ // arithmetic of SOB's evaluate.py (f1_from_counts, token_f1).
15
+ export class PrecisionRecallForm {
16
+ of({ hits, stated, expected }, beta) {
17
+ if (hits === 0)
18
+ return 0;
19
+ const beta2 = beta ** 2;
20
+ const precision = hits / stated;
21
+ const recall = hits / expected;
22
+ return ((1 + beta2) * precision * recall) / (beta2 * precision + recall);
23
+ }
24
+ }
25
+ //# sourceMappingURL=f-measure-form.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"f-measure-form.js","sourceRoot":"","sources":["../../src/aggregation/f-measure-form.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAQnC,4EAA4E;AAC5E,4EAA4E;AAC5E,yEAAyE;AACzE,cAAc;AACd,MAAM,OAAO,UAAU;IACrB,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAU,EAAE,IAAY;QACjD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC;QACxB,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC;IACxE,CAAC;CACF;AAED,wEAAwE;AACxE,2EAA2E;AAC3E,8DAA8D;AAC9D,MAAM,OAAO,mBAAmB;IAC9B,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAU,EAAE,IAAY;QACjD,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC;QACxB,MAAM,SAAS,GAAG,IAAI,GAAG,MAAM,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,GAAG,QAAQ,CAAC;QAC/B,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,GAAG,MAAM,CAAC,CAAC;IAC3E,CAAC;CACF"}
@@ -0,0 +1,12 @@
1
+ import type { Counts } from './counts.ts';
2
+ import type { FMeasureForm } from './f-measure-form.ts';
3
+ import { Ratio } from './ratio.ts';
4
+ export declare class FMeasureRatio extends Ratio {
5
+ #private;
6
+ constructor(counts: Counts, { beta, form }: {
7
+ readonly beta: number;
8
+ readonly form: FMeasureForm;
9
+ });
10
+ get value(): number;
11
+ }
12
+ //# sourceMappingURL=f-measure-ratio.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"f-measure-ratio.d.ts","sourceRoot":"","sources":["../../src/aggregation/f-measure-ratio.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMnC,qBAAa,aAAc,SAAQ,KAAK;;gBAMpC,MAAM,EAAE,MAAM,EACd,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAA;KAAE;IASxE,IAAa,KAAK,IAAI,MAAM,CAE3B;CACF"}
@@ -0,0 +1,21 @@
1
+ import { Ratio } from "./ratio.js";
2
+ // The ratio of an F-measure: its part is (1 + β²)·hits and its whole is
3
+ // β²·expected + stated, so many of them add up to the micro F-measure. Its
4
+ // value is the value of its form, which can differ from part / whole in the
5
+ // last bit (ADR 0003).
6
+ export class FMeasureRatio extends Ratio {
7
+ #counts;
8
+ #beta;
9
+ #form;
10
+ constructor(counts, { beta, form }) {
11
+ const beta2 = beta ** 2;
12
+ super((1 + beta2) * counts.hits, beta2 * counts.expected + counts.stated);
13
+ this.#counts = counts;
14
+ this.#beta = beta;
15
+ this.#form = form;
16
+ }
17
+ get value() {
18
+ return this.#form.of(this.#counts, this.#beta);
19
+ }
20
+ }
21
+ //# sourceMappingURL=f-measure-ratio.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"f-measure-ratio.js","sourceRoot":"","sources":["../../src/aggregation/f-measure-ratio.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,wEAAwE;AACxE,2EAA2E;AAC3E,4EAA4E;AAC5E,uBAAuB;AACvB,MAAM,OAAO,aAAc,SAAQ,KAAK;IAC7B,OAAO,CAAS;IAChB,KAAK,CAAS;IACd,KAAK,CAAe;IAE7B,YACE,MAAc,EACd,EAAE,IAAI,EAAE,IAAI,EAA0D;QAEtE,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC;QACxB,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1E,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,IAAa,KAAK;QAChB,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACjD,CAAC;CACF"}
@@ -1,6 +1,7 @@
1
1
  import { Ratio } from './ratio.ts';
2
2
  export declare class RatioOfSums {
3
3
  #private;
4
+ static of(ratios: Iterable<Ratio>): RatioOfSums;
4
5
  add(ratio: Ratio): void;
5
6
  get ratio(): Ratio;
6
7
  get value(): number;
@@ -1 +1 @@
1
- {"version":3,"file":"ratio-of-sums.d.ts","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMnC,qBAAa,WAAW;;IAItB,GAAG,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAKvB,IAAI,KAAK,IAAI,KAAK,CAEjB;IAED,IAAI,KAAK,IAAI,MAAM,CAElB;CACF"}
1
+ {"version":3,"file":"ratio-of-sums.d.ts","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMnC,qBAAa,WAAW;;IAKtB,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,WAAW;IAM/C,GAAG,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAKvB,IAAI,KAAK,IAAI,KAAK,CAEjB;IAED,IAAI,KAAK,IAAI,MAAM,CAElB;CACF"}
@@ -7,6 +7,13 @@ import { Ratio } from "./ratio.js";
7
7
  export class RatioOfSums {
8
8
  #parts = new NeumaierSum();
9
9
  #wholes = new NeumaierSum();
10
+ // The ratio of sums of the ratios, in their order.
11
+ static of(ratios) {
12
+ const sum = new RatioOfSums();
13
+ for (const ratio of ratios)
14
+ sum.add(ratio);
15
+ return sum;
16
+ }
10
17
  add(ratio) {
11
18
  this.#parts.add(ratio.part);
12
19
  this.#wholes.add(ratio.whole);
@@ -1 +1 @@
1
- {"version":3,"file":"ratio-of-sums.js","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,0EAA0E;AAC1E,sEAAsE;AACtE,2EAA2E;AAC3E,sCAAsC;AACtC,MAAM,OAAO,WAAW;IACb,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;IAC3B,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAErC,GAAG,CAAC,KAAY;QACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;IAC1B,CAAC;CACF"}
1
+ {"version":3,"file":"ratio-of-sums.js","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,0EAA0E;AAC1E,sEAAsE;AACtE,2EAA2E;AAC3E,sCAAsC;AACtC,MAAM,OAAO,WAAW;IACb,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;IAC3B,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAErC,mDAAmD;IACnD,MAAM,CAAC,EAAE,CAAC,MAAuB;QAC/B,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,KAAK,MAAM,KAAK,IAAI,MAAM;YAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3C,OAAO,GAAG,CAAC;IACb,CAAC;IAED,GAAG,CAAC,KAAY;QACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;IAC1B,CAAC;CACF"}
@@ -0,0 +1,7 @@
1
+ import { Counts } from '../aggregation/counts.ts';
2
+ import type { Matches } from '../scorers/matches.ts';
3
+ import type { LabelComparison } from './label-comparison.ts';
4
+ export declare class AllClassMatches implements Matches<LabelComparison> {
5
+ counts({ stated, expected }: LabelComparison): Counts;
6
+ }
7
+ //# sourceMappingURL=all-class-matches.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"all-class-matches.d.ts","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAK7D,qBAAa,eAAgB,YAAW,OAAO,CAAC,eAAe,CAAC;IAC9D,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,eAAe,GAAG,MAAM;CAOtD"}
@@ -0,0 +1,14 @@
1
+ import { Counts } from "../aggregation/counts.js";
2
+ // The counts of every class in one answer: the hits are the labels that both
3
+ // sides have, the stated and the expected are the labels of each side.
4
+ // Pooled over the answers, they are scikit-learn's micro counts.
5
+ export class AllClassMatches {
6
+ counts({ stated, expected }) {
7
+ return new Counts({
8
+ hits: stated.intersection(expected).size,
9
+ stated: stated.size,
10
+ expected: expected.size,
11
+ });
12
+ }
13
+ }
14
+ //# sourceMappingURL=all-class-matches.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"all-class-matches.js","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIlD,6EAA6E;AAC7E,uEAAuE;AACvE,iEAAiE;AACjE,MAAM,OAAO,eAAe;IAC1B,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAmB;QAC1C,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI;YACxC,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,QAAQ,EAAE,QAAQ,CAAC,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;CACF"}
@@ -0,0 +1,11 @@
1
+ import type { Counts } from '../aggregation/counts.ts';
2
+ import type { Label } from './label.ts';
3
+ export type Classes = {
4
+ readonly selected: ReadonlyMap<Label, Counts>;
5
+ readonly present: ReadonlyMap<Label, Counts>;
6
+ readonly multilabel: boolean;
7
+ };
8
+ export interface Average {
9
+ of(classes: Classes, score: (counts: Counts) => number): number;
10
+ }
11
+ //# sourceMappingURL=average.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"average.d.ts","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,MAAM,MAAM,OAAO,GAAG;IACpB,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC9C,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;CAC9B,CAAC;AAKF,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,CAAC;CACjE"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=average.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"average.js","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":""}