@zukhruf/evals 0.8.3 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -3
- package/README.md +152 -79
- package/dist/aggregation/f-measure-form.d.ts +11 -0
- package/dist/aggregation/f-measure-form.d.ts.map +1 -0
- package/dist/aggregation/f-measure-form.js +25 -0
- package/dist/aggregation/f-measure-form.js.map +1 -0
- package/dist/aggregation/f-measure-ratio.d.ts +12 -0
- package/dist/aggregation/f-measure-ratio.d.ts.map +1 -0
- package/dist/aggregation/f-measure-ratio.js +21 -0
- package/dist/aggregation/f-measure-ratio.js.map +1 -0
- package/dist/aggregation/ratio-of-sums.d.ts +1 -0
- package/dist/aggregation/ratio-of-sums.d.ts.map +1 -1
- package/dist/aggregation/ratio-of-sums.js +7 -0
- package/dist/aggregation/ratio-of-sums.js.map +1 -1
- package/dist/classification/all-class-matches.d.ts +7 -0
- package/dist/classification/all-class-matches.d.ts.map +1 -0
- package/dist/classification/all-class-matches.js +14 -0
- package/dist/classification/all-class-matches.js.map +1 -0
- package/dist/classification/average.d.ts +11 -0
- package/dist/classification/average.d.ts.map +1 -0
- package/dist/classification/average.js +2 -0
- package/dist/classification/average.js.map +1 -0
- package/dist/classification/binary-average.d.ts +9 -0
- package/dist/classification/binary-average.d.ts.map +1 -0
- package/dist/classification/binary-average.js +20 -0
- package/dist/classification/binary-average.js.map +1 -0
- package/dist/classification/brier-score.d.ts +13 -0
- package/dist/classification/brier-score.d.ts.map +1 -0
- package/dist/classification/brier-score.js +52 -0
- package/dist/classification/brier-score.js.map +1 -0
- package/dist/classification/class-counts.d.ts +39 -0
- package/dist/classification/class-counts.d.ts.map +1 -0
- package/dist/classification/class-counts.js +159 -0
- package/dist/classification/class-counts.js.map +1 -0
- package/dist/classification/class-matches.d.ts +10 -0
- package/dist/classification/class-matches.d.ts.map +1 -0
- package/dist/classification/class-matches.js +21 -0
- package/dist/classification/class-matches.js.map +1 -0
- package/dist/classification/confusion-matrix.d.ts +9 -0
- package/dist/classification/confusion-matrix.d.ts.map +1 -0
- package/dist/classification/confusion-matrix.js +18 -0
- package/dist/classification/confusion-matrix.js.map +1 -0
- package/dist/classification/index.d.ts +21 -0
- package/dist/classification/index.d.ts.map +1 -0
- package/dist/classification/index.js +19 -0
- package/dist/classification/index.js.map +1 -0
- package/dist/classification/label-case.d.ts +11 -0
- package/dist/classification/label-case.d.ts.map +1 -0
- package/dist/classification/label-case.js +12 -0
- package/dist/classification/label-case.js.map +1 -0
- package/dist/classification/label-comparer.d.ts +8 -0
- package/dist/classification/label-comparer.d.ts.map +1 -0
- package/dist/classification/label-comparer.js +44 -0
- package/dist/classification/label-comparer.js.map +1 -0
- package/dist/classification/label-comparison.d.ts +12 -0
- package/dist/classification/label-comparison.d.ts.map +1 -0
- package/dist/classification/label-comparison.js +23 -0
- package/dist/classification/label-comparison.js.map +1 -0
- package/dist/classification/label-config-case.d.ts +6 -0
- package/dist/classification/label-config-case.d.ts.map +1 -0
- package/dist/classification/label-config-case.js +13 -0
- package/dist/classification/label-config-case.js.map +1 -0
- package/dist/classification/label-vars-case.d.ts +6 -0
- package/dist/classification/label-vars-case.d.ts.map +1 -0
- package/dist/classification/label-vars-case.js +17 -0
- package/dist/classification/label-vars-case.js.map +1 -0
- package/dist/classification/label.d.ts +4 -0
- package/dist/classification/label.d.ts.map +1 -0
- package/dist/classification/label.js +16 -0
- package/dist/classification/label.js.map +1 -0
- package/dist/classification/labels-match.d.ts +7 -0
- package/dist/classification/labels-match.d.ts.map +1 -0
- package/dist/classification/labels-match.js +13 -0
- package/dist/classification/labels-match.js.map +1 -0
- package/dist/classification/macro-average.d.ts +6 -0
- package/dist/classification/macro-average.d.ts.map +1 -0
- package/dist/classification/macro-average.js +15 -0
- package/dist/classification/macro-average.js.map +1 -0
- package/dist/classification/micro-average.d.ts +6 -0
- package/dist/classification/micro-average.d.ts.map +1 -0
- package/dist/classification/micro-average.js +16 -0
- package/dist/classification/micro-average.js.map +1 -0
- package/dist/classification/per-class.d.ts +12 -0
- package/dist/classification/per-class.d.ts.map +1 -0
- package/dist/classification/per-class.js +17 -0
- package/dist/classification/per-class.js.map +1 -0
- package/dist/classification/probability-comparer.d.ts +8 -0
- package/dist/classification/probability-comparer.d.ts.map +1 -0
- package/dist/classification/probability-comparer.js +35 -0
- package/dist/classification/probability-comparer.js.map +1 -0
- package/dist/classification/probability-comparison.d.ts +10 -0
- package/dist/classification/probability-comparison.d.ts.map +1 -0
- package/dist/classification/probability-comparison.js +11 -0
- package/dist/classification/probability-comparison.js.map +1 -0
- package/dist/classification/read-labels.d.ts +3 -0
- package/dist/classification/read-labels.d.ts.map +1 -0
- package/dist/classification/read-labels.js +11 -0
- package/dist/classification/read-labels.js.map +1 -0
- package/dist/classification/weighted-average.d.ts +6 -0
- package/dist/classification/weighted-average.d.ts.map +1 -0
- package/dist/classification/weighted-average.js +20 -0
- package/dist/classification/weighted-average.js.map +1 -0
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/promptfoo/index.d.ts +1 -0
- package/dist/promptfoo/index.d.ts.map +1 -1
- package/dist/promptfoo/index.js +1 -0
- package/dist/promptfoo/index.js.map +1 -1
- package/dist/promptfoo/scorer-assertion.d.ts +18 -0
- package/dist/promptfoo/scorer-assertion.d.ts.map +1 -0
- package/dist/{structured/promptfoo → promptfoo}/scorer-assertion.js +5 -11
- package/dist/promptfoo/scorer-assertion.js.map +1 -0
- package/dist/scorers/f-beta.d.ts +13 -0
- package/dist/scorers/f-beta.d.ts.map +1 -0
- package/dist/scorers/f-beta.js +22 -0
- package/dist/scorers/f-beta.js.map +1 -0
- package/dist/scorers/f1.d.ts +6 -6
- package/dist/scorers/f1.d.ts.map +1 -1
- package/dist/scorers/f1.js +5 -9
- package/dist/scorers/f1.js.map +1 -1
- package/dist/scorers/penalized.d.ts.map +1 -1
- package/dist/scorers/penalized.js +2 -1
- package/dist/scorers/penalized.js.map +1 -1
- package/dist/scoring/bootstrap.js +3 -5
- package/dist/scoring/bootstrap.js.map +1 -1
- package/dist/scoring/comparer.d.ts +5 -0
- package/dist/scoring/comparer.d.ts.map +1 -0
- package/dist/scoring/comparer.js +2 -0
- package/dist/scoring/comparer.js.map +1 -0
- package/dist/scoring/grouped.d.ts +9 -0
- package/dist/scoring/grouped.d.ts.map +1 -0
- package/dist/scoring/grouped.js +20 -0
- package/dist/scoring/grouped.js.map +1 -0
- package/dist/scoring/scorer-set.d.ts +13 -0
- package/dist/scoring/scorer-set.d.ts.map +1 -0
- package/dist/scoring/scorer-set.js +55 -0
- package/dist/scoring/scorer-set.js.map +1 -0
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts +4 -1
- package/dist/structured/comparison/comparators/token-overlap-at-least.d.ts.map +1 -1
- package/dist/structured/comparison/comparators/token-overlap-at-least.js +7 -4
- package/dist/structured/comparison/comparators/token-overlap-at-least.js.map +1 -1
- package/dist/structured/comparison/json-value.d.ts +1 -0
- package/dist/structured/comparison/json-value.d.ts.map +1 -1
- package/dist/structured/comparison/json-value.js +11 -2
- package/dist/structured/comparison/json-value.js.map +1 -1
- package/dist/structured/comparison/tree-walk.d.ts +5 -5
- package/dist/structured/comparison/tree-walk.d.ts.map +1 -1
- package/dist/structured/comparison/tree-walk.js +8 -4
- package/dist/structured/comparison/tree-walk.js.map +1 -1
- package/dist/structured/cpython/python-text.d.ts +0 -1
- package/dist/structured/cpython/python-text.d.ts.map +1 -1
- package/dist/structured/cpython/python-text.js +1 -18
- package/dist/structured/cpython/python-text.js.map +1 -1
- package/dist/structured/index.d.ts +0 -2
- package/dist/structured/index.d.ts.map +1 -1
- package/dist/structured/index.js +0 -2
- package/dist/structured/index.js.map +1 -1
- package/dist/structured/scorers/token-f1.d.ts +3 -1
- package/dist/structured/scorers/token-f1.d.ts.map +1 -1
- package/dist/structured/scorers/token-f1.js +8 -5
- package/dist/structured/scorers/token-f1.js.map +1 -1
- package/dist/structured/scoring/per-field.d.ts +3 -6
- package/dist/structured/scoring/per-field.d.ts.map +1 -1
- package/dist/structured/scoring/per-field.js +8 -17
- package/dist/structured/scoring/per-field.js.map +1 -1
- package/package.json +18 -5
- package/dist/aggregation/f1-ratio.d.ts +0 -8
- package/dist/aggregation/f1-ratio.d.ts.map +0 -1
- package/dist/aggregation/f1-ratio.js +0 -24
- package/dist/aggregation/f1-ratio.js.map +0 -1
- package/dist/structured/promptfoo/scorer-assertion.d.ts +0 -22
- package/dist/structured/promptfoo/scorer-assertion.d.ts.map +0 -1
- package/dist/structured/promptfoo/scorer-assertion.js.map +0 -1
- package/dist/structured/scoring/scorer-set.d.ts +0 -15
- package/dist/structured/scoring/scorer-set.d.ts.map +0 -1
- package/dist/structured/scoring/scorer-set.js +0 -60
- package/dist/structured/scoring/scorer-set.js.map +0 -1
package/NOTICE
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
@zukhruf/
|
|
1
|
+
@zukhruf/evals contains a TypeScript port of the scorer of the
|
|
2
2
|
Structured Output Benchmark (SOB): evaluate.py at commit da785a8 of
|
|
3
3
|
https://github.com/interfazeai/sob. That code has this licence:
|
|
4
4
|
|
|
@@ -24,7 +24,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
|
24
24
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
25
|
SOFTWARE.
|
|
26
26
|
|
|
27
|
-
@zukhruf/
|
|
27
|
+
@zukhruf/evals contains test cases from the JSON-Schema-Test-Suite:
|
|
28
28
|
tests/draft2020-12/optional/ecmascript-regex.json and
|
|
29
29
|
tests/draft7/optional/ecmascript-regex.json at commit
|
|
30
30
|
245a45abc90190c0bff04d97e6d2b541b4bb9f12 of
|
|
@@ -53,7 +53,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
|
53
53
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
54
54
|
THE SOFTWARE.
|
|
55
55
|
|
|
56
|
-
@zukhruf/
|
|
56
|
+
@zukhruf/evals contains the JSON Schema meta-schemas of draft 7 and
|
|
57
57
|
draft 2020-12 (src/schema/draft-07-schema.ts and
|
|
58
58
|
src/schema/draft-2020-12-schema.ts), copied from jsonschema-specifications
|
|
59
59
|
2025.9.1 (https://github.com/python-jsonschema/jsonschema-specifications). That
|
package/README.md
CHANGED
|
@@ -6,18 +6,18 @@ The answers of evals have different shapes: a structured record, a label, a rank
|
|
|
6
6
|
|
|
7
7
|
## The parts
|
|
8
8
|
|
|
9
|
-
| Import path
|
|
10
|
-
|
|
|
11
|
-
| `@zukhruf/evals`
|
|
12
|
-
| `@zukhruf/evals/promptfoo`
|
|
13
|
-
| `@zukhruf/evals/structured`
|
|
14
|
-
| `@zukhruf/evals/classification`
|
|
15
|
-
| `@zukhruf/evals/ranking` (later)
|
|
16
|
-
| `@zukhruf/evals/clustering` (later)
|
|
9
|
+
| Import path | Job |
|
|
10
|
+
| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------- |
|
|
11
|
+
| `@zukhruf/evals` | The core: scorers, ratios, gates, scorer sets, scorecards, totals and bootstrap intervals. |
|
|
12
|
+
| `@zukhruf/evals/promptfoo` | The adapter that makes a scorer or a scorer set a promptfoo assertion. |
|
|
13
|
+
| `@zukhruf/evals/structured` | Compares a structured value with the value that a person labelled. |
|
|
14
|
+
| `@zukhruf/evals/classification` | Compares the stated labels of an answer with its expected labels, as scikit-learn's `sklearn.metrics` does (ADR 0003). |
|
|
15
|
+
| `@zukhruf/evals/ranking` (later) | Compares a ranked list with judged items, as trec_eval does. |
|
|
16
|
+
| `@zukhruf/evals/clustering` (later) | Compares a partition with a labelled partition, as the CoNLL reference scorer does. |
|
|
17
17
|
|
|
18
18
|
A subpath imports only the core. It never imports another subpath.
|
|
19
19
|
|
|
20
|
-
The
|
|
20
|
+
The core holds the parts that all families share: the ratios, the scorers, the penalties, `Comparer`, `Grouped`, `ScorerSet`, `Totals` and `Bootstrap`. A part that reads an input has a type parameter for that input. So a scorer of one family cannot read the input of another family (ADR 0002). `./promptfoo` holds `ScorerAssertion`. The families are `./structured` and `./classification`.
|
|
21
21
|
|
|
22
22
|
## Score one answer
|
|
23
23
|
|
|
@@ -41,6 +41,11 @@ new Precision(new ClassMatches('home')).score(comparison); // Ratio { part: 0, w
|
|
|
41
41
|
|
|
42
42
|
A `Ratio` is a part and a whole. Its value is the part divided by the whole, and 0 when the whole is 0. `Precision`, `Recall`, `F1` and `FBeta` read the counts of a `Matches`: the hits, the stated and the expected. Each family gives its own matches, and these scorers take any of them.
|
|
43
43
|
|
|
44
|
+
The ratio of `new FBeta(matches, { beta, form })` has the part (1 + β²) × hits and the whole β² × expected + stated, so many of them add up to the micro F-measure. `F1` is `FBeta` with β = 1. The form gives the value (ADR 0003):
|
|
45
|
+
|
|
46
|
+
- `new CountsForm()` is the default. It is scikit-learn's arithmetic, and its value is the part divided by the whole: 0 when the whole is 0. So 1 hit of 5 stated and 1 expected gives 1/3.
|
|
47
|
+
- `new PrecisionRecallForm()` is (1 + β²)·P·R / (β²·P + R): 0 when there are no hits. SOB's evaluate.py uses it. The same counts give 0.33333333333333337.
|
|
48
|
+
|
|
44
49
|
## Combine scorers
|
|
45
50
|
|
|
46
51
|
A penalty is a factor in [0, 1] for one input, as BLEU's brevity penalty is. The factor 1 is no penalty. A penalty is also a scorer: its ratio is `Ratio(factor, 1)`. A gate is the penalty whose factor is 0 or 1. A gate says what it checks (`holds`) and why it fails (`explain`).
|
|
@@ -53,9 +58,9 @@ A penalty is a factor in [0, 1] for one input, as BLEU's brevity penalty is. The
|
|
|
53
58
|
|
|
54
59
|
## Score with many scorers at once
|
|
55
60
|
|
|
56
|
-
A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. The family
|
|
61
|
+
A `ScorerSet` holds labelled scorers and gives a `Scorecard`: passed, reason, checks and one named ratio for each label. It passes when each of its gates holds. Each gate gives a check, whose `name` is the label of the gate. The `comparer` of the family is required, because the core cannot know a family: `new ScorerSet(scorers, { comparer: new LabelComparer() })`. It makes the input once, and each scorer reads it.
|
|
57
62
|
|
|
58
|
-
A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core
|
|
63
|
+
A grouped scorer gives one ratio for each group of an input, under the label `${group} ${label}`: `PerField` gives one for each field in `./structured`, and `PerClass` gives one for each class in `./classification`. Both extend `Grouped` of the core: a subclass gives its groups with `groups(input)`, and `ratios` drops a group whose whole is 0. So a `ScorerSet` holds them beside plain scorers (Inspect AI's `grouped()` applies a metric to each group in the same way).
|
|
59
64
|
|
|
60
65
|
```ts
|
|
61
66
|
import { Recall, ScorerSet } from '@zukhruf/evals';
|
|
@@ -81,6 +86,10 @@ const pageKinds = new ScorerSet(
|
|
|
81
86
|
- A plain scorer gives its counts, such as Ratio(2, 3). `Totals` then gives the micro average.
|
|
82
87
|
- A `SampleAveraged` scorer gives its value over 1. `Totals` then gives the samples average: the mean of the values of the answers.
|
|
83
88
|
|
|
89
|
+
`Totals` adds with CPython's compensated `sum()`. A sum of integer weights is exact, so its means and accuracies equal scikit-learn's for integer weights. A sum of float weights can differ from numpy's pairwise sum in the last bit. For float weights, use `ClassCounts` and `BrierScore.mean` of `./classification`.
|
|
90
|
+
|
|
91
|
+
`RatioOfSums.of(ratios)` adds a list of ratios in one call.
|
|
92
|
+
|
|
84
93
|
`Bootstrap` gives the interval of one name over the scorecards, with `@zukhruf/statistics`. You give it the number of resamples, the random index and the method of the interval.
|
|
85
94
|
|
|
86
95
|
## Compare structured values
|
|
@@ -91,67 +100,92 @@ It needs two optional peers: `npm install @zukhruf/evals @cfworker/json-schema m
|
|
|
91
100
|
|
|
92
101
|
## Score labels
|
|
93
102
|
|
|
94
|
-
|
|
103
|
+
`./classification` compares the stated labels of an answer with its expected labels. One label is a set of one label, so one model serves multiclass answers (one class) and multilabel answers (a set of classes). The arithmetic follows scikit-learn 1.9.1, bit for bit (ADR 0003). The golden tests compare each number with the output of `src/classification/fixtures/sklearn/generate.py`.
|
|
95
104
|
|
|
96
|
-
|
|
105
|
+
It needs one optional peer: `npm install @zukhruf/evals @stdlib/blas-ext-base-gsumpw`. That package gives numpy's pairwise sum.
|
|
97
106
|
|
|
98
107
|
### For each answer
|
|
99
108
|
|
|
100
|
-
|
|
109
|
+
A case holds the expected labels and the weight: `new LabelCase({ expected, weight })`. `expected` is a label or a list of labels. `LabelVarsCase` reads `vars.expected` and `vars.weight` of a test (a test without a weight weighs 1). `LabelConfigCase` reads `config.expected` of an assertion, with the weight 1.
|
|
110
|
+
|
|
111
|
+
`LabelComparer` reads the stated labels of an answer and gives a `LabelComparison`. It reads a label or a list of labels as they are. It reads text in this order:
|
|
112
|
+
|
|
113
|
+
1. A JSON array of labels gives those labels.
|
|
114
|
+
2. A number gives a number label, but only when each expected label of the case is a number. So `"7"` matches the label `7` and the label `"7"`.
|
|
115
|
+
3. Other text is one string label, without its white space at the start and the end.
|
|
116
|
+
|
|
117
|
+
`comparison.only(label)` gives the comparison of one class: each side keeps the label when it has it.
|
|
118
|
+
|
|
119
|
+
The matches of a comparison count one answer:
|
|
101
120
|
|
|
102
121
|
| Matches | Hits | Stated | Expected |
|
|
103
122
|
| ------------------------- | -------------------------------- | ------------------------ | ---------------------- |
|
|
104
123
|
| `new ClassMatches(label)` | 1 when both sides have the label | 1 when the answer has it | 1 when the case has it |
|
|
105
124
|
| `new AllClassMatches()` | the labels that both sides have | the stated labels | the expected labels |
|
|
106
125
|
|
|
107
|
-
`LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `
|
|
126
|
+
`LabelsMatch` is a gate: it holds when the stated labels are the expected labels. `new PerClass(scorer, { labels })` gives one ratio for each class, under the label `${class} ${label}`. Without `labels`, the classes are the labels of both sides of the answer, in the order of numpy's `unique()` (numbers by value, strings by code point).
|
|
127
|
+
|
|
128
|
+
`Totals` of these ratios gives scikit-learn's numbers, bit for bit, for integer weights:
|
|
108
129
|
|
|
109
|
-
|
|
130
|
+
- `ClassMatches` gives the precision, recall and F1 of each class (`average=None`).
|
|
131
|
+
- `AllClassMatches` gives the micro average.
|
|
132
|
+
- `LabelsMatch` gives `accuracy_score` (for multilabel answers, the subset accuracy).
|
|
133
|
+
|
|
134
|
+
So a promptfoo run gives the micro precision, recall and F1, and the score of each class, in its column headers.
|
|
110
135
|
|
|
111
136
|
### For a whole run
|
|
112
137
|
|
|
113
|
-
A macro or weighted average is not a ratio of sums: it
|
|
138
|
+
A macro average or a weighted average is not a ratio of sums: it needs the score of each class first. `ClassCounts` keeps the counts of each class: the true positives (hits), the predicted (stated) and the true (expected). It adds the weight of each answer in the order of the answers, as numpy's `bincount` does. torchmetrics does the same with `update()` and `compute()`.
|
|
114
139
|
|
|
115
140
|
```ts
|
|
116
141
|
import {
|
|
117
142
|
ClassCounts,
|
|
118
|
-
|
|
143
|
+
LabelCase,
|
|
144
|
+
LabelComparer,
|
|
119
145
|
MacroAverage,
|
|
120
146
|
WeightedAverage,
|
|
121
147
|
} from '@zukhruf/evals/classification';
|
|
122
148
|
|
|
123
|
-
const counts =
|
|
124
|
-
|
|
149
|
+
const counts = ClassCounts.of({
|
|
150
|
+
expected: ['a', 'b', 'b'],
|
|
151
|
+
stated: ['a', 'a', 'b'],
|
|
152
|
+
});
|
|
125
153
|
|
|
126
154
|
counts.accuracy(); // accuracy_score
|
|
127
155
|
counts.fBeta({ beta: 1, average: new MacroAverage() }); // f1_score(average='macro')
|
|
128
156
|
counts.precision({ average: new WeightedAverage() }); // precision_score(average='weighted')
|
|
129
|
-
counts.
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
});
|
|
134
|
-
|
|
157
|
+
counts.confusionMatrix().rows(); // confusion_matrix, labels in sorted order
|
|
158
|
+
|
|
159
|
+
// The same counts, one answer at a time:
|
|
160
|
+
const run = new ClassCounts({ multilabel: false });
|
|
161
|
+
const testCase = new LabelCase({ expected: 'a', weight: 1 });
|
|
162
|
+
run.add(new LabelComparer().compare('a', testCase), testCase.weight);
|
|
135
163
|
```
|
|
136
164
|
|
|
137
|
-
- `
|
|
138
|
-
- `
|
|
139
|
-
- `
|
|
140
|
-
- `
|
|
165
|
+
- `ClassCounts.of({ expected, stated, weights })` builds multiclass counts from two arrays of labels. `weights` is optional, and an answer without a weight weighs 1.
|
|
166
|
+
- `ClassCounts.ofIndicators({ expected, stated, weights })` builds multilabel counts from two label indicator matrices. Each column is a class, also when no answer has it.
|
|
167
|
+
- `new ClassCounts({ multilabel })` starts empty, and `add(comparison, weight)` adds one answer. Multiclass counts refuse an answer with no label or more than one label on a side.
|
|
168
|
+
- `average` is a Strategy: `new MicroAverage()`, `new MacroAverage()`, `new WeightedAverage()` or `new BinaryAverage(positiveLabel)`. `Average.of(classes, score)` gives one number from the score of each class. The micro average adds the counts with numpy's pairwise sum. The macro average is numpy's `nanmean`. The weighted average is scikit-learn's `_nanaverage`: it drops each NaN score with its weight, and it gives the plain mean when the weights add up to 0. `BinaryAverage` throws when scikit-learn raises: for multilabel counts, for more than two classes, and for a positive label that is not one of two classes. The samples average is a `SampleAveraged` scorer, because it needs the value of each answer.
|
|
169
|
+
- `labels` selects the classes to score, in its order. By default it is every class, in sorted order.
|
|
170
|
+
- `zeroDivision` is the score of a class whose denominator is 0: 0 (the default, as scikit-learn's `"warn"` gives), 1 or `NaN`. A macro average and a weighted average skip a `NaN` score.
|
|
171
|
+
- `form` is the arithmetic of the F-measure (ADR 0003). `new CountsForm()` is scikit-learn's and the default. It also gives the bits of TREC's F0.5 for integer counts. `new PrecisionRecallForm()` is van Rijsbergen's and SOB's.
|
|
172
|
+
- `confusionMatrix()` throws for multilabel counts, as scikit-learn raises.
|
|
141
173
|
|
|
142
174
|
### Probabilities
|
|
143
175
|
|
|
144
|
-
`
|
|
176
|
+
`ProbabilityComparer` reads the stated probability of each class of an answer and gives a `ProbabilityComparison`. The answer is a `Map`, a plain object or the JSON text of an object. A key is a number label when the expected label is a number. A probability outside [0, 1] throws a `RangeError`. The case has one expected label.
|
|
177
|
+
|
|
178
|
+
`BrierScore` follows scikit-learn's `brier_score_loss`. For one answer, it adds the squares of (1 for the expected label, else 0, minus the probability) over the classes in sorted order, with numpy's pairwise sum. For two classes, it multiplies the sum by 0.5. `new BrierScore({ positiveLabel })` reads only the probability p of the positive label, and the two classes are 1 − p and p. Its ratio is the score of one answer over 1. `brierScore.mean(comparisons, weights)` gives the score of a whole run with numpy's pairwise sum, bit for bit.
|
|
145
179
|
|
|
146
180
|
## Use it with promptfoo
|
|
147
181
|
|
|
148
|
-
|
|
182
|
+
`ScorerAssertion` is in `./promptfoo`. It takes the comparer and the case reader of a family. The sample below scores labels. [docs/structured.md](docs/structured.md) has a structured sample.
|
|
149
183
|
|
|
150
184
|
`PromptfooAssertion` adapts a scorer set to a per-row assertion. Each ratio becomes a named score with its whole as the weight, so the column header shows the same ratio of sums as `Totals`. It reads the case from the config of the assertion.
|
|
151
185
|
|
|
152
186
|
`ScorerAssertion` holds one scorer with its `weight` (how much it counts in the score of the test) and its `threshold` (when it passes). Both numbers are required. It reads the case from the vars of the test.
|
|
153
187
|
|
|
154
|
-
promptfoo stores the config of a run as JSON, and `--resume` and `--retry-errors` build the run from that stored config. A function does not survive JSON, so a config holds no scorer assertion. An `AssertionModule` holds the scorer assertions by name, in a module that promptfoo loads. Its `assert(...names)` gives JSON assertions whose `value` is the file reference `file://<module>:<name>`. promptfoo loads the function from the default export of the module when it grades, on the first run and on a resumed run (structured
|
|
188
|
+
promptfoo stores the config of a run as JSON, and `--resume` and `--retry-errors` build the run from that stored config. A function does not survive JSON, so a config holds no scorer assertion. An `AssertionModule` holds the scorer assertions by name, in a module that promptfoo loads. Its `assert(...names)` gives JSON assertions whose `value` is the file reference `file://<module>:<name>`. promptfoo loads the function from the default export of the module when it grades, on the first run and on a resumed run (structured ADR 0013 in docs/adr/structured).
|
|
155
189
|
|
|
156
190
|
```ts
|
|
157
191
|
// page-kinds/scorers.ts: the module that promptfoo loads
|
|
@@ -313,6 +347,7 @@ export interface Comparer<Case extends Weighted, Input> {
|
|
|
313
347
|
}
|
|
314
348
|
|
|
315
349
|
export type Check = {
|
|
350
|
+
readonly name: string;
|
|
316
351
|
readonly passed: boolean;
|
|
317
352
|
readonly score: number;
|
|
318
353
|
readonly reason: string;
|
|
@@ -330,7 +365,11 @@ export interface ScorecardScorer<Case extends Weighted> {
|
|
|
330
365
|
}
|
|
331
366
|
/** Gives one ratio for each group of an input, such as each field or each class. */
|
|
332
367
|
export abstract class Grouped<Input> {
|
|
333
|
-
|
|
368
|
+
constructor(scorer: Scorer<Input>);
|
|
369
|
+
/** The groups of an input (Template Method). */
|
|
370
|
+
protected abstract groups(input: Input): Iterable<[string, Input]>;
|
|
371
|
+
/** The ratio of the scorer on each group, under `${group} ${label}`. A group whose whole is 0 gives no ratio. */
|
|
372
|
+
ratios(input: Input, label: string): [string, Ratio][];
|
|
334
373
|
}
|
|
335
374
|
export class ScorerSet<
|
|
336
375
|
Case extends Weighted,
|
|
@@ -344,6 +383,8 @@ export class ScorerSet<
|
|
|
344
383
|
|
|
345
384
|
/** Adds ratios: the sum of the parts divided by the sum of the wholes. */
|
|
346
385
|
export class RatioOfSums {
|
|
386
|
+
/** The ratio of sums of a list of ratios, in one call. */
|
|
387
|
+
static of(ratios: Iterable<Ratio>): RatioOfSums;
|
|
347
388
|
add(ratio: Ratio): void;
|
|
348
389
|
get ratio(): Ratio;
|
|
349
390
|
get value(): number;
|
|
@@ -404,43 +445,55 @@ export type FileAssertion = {
|
|
|
404
445
|
readonly threshold: number;
|
|
405
446
|
readonly value: string; // file://<module>:<name>
|
|
406
447
|
};
|
|
407
|
-
/** The
|
|
448
|
+
/** The vars of the test that promptfoo gives a javascript assertion of a shared defaultTest. */
|
|
449
|
+
export type VarsContext = {
|
|
450
|
+
readonly vars: Readonly<Record<string, unknown>>;
|
|
451
|
+
};
|
|
452
|
+
/** What an assertion module needs of an assertion. A ScorerAssertion is one. */
|
|
453
|
+
export type ModuleAssertion = {
|
|
454
|
+
readonly metric?: string | undefined;
|
|
455
|
+
readonly weight: number;
|
|
456
|
+
readonly threshold: number;
|
|
457
|
+
readonly value: (output: string, context: VarsContext) => number;
|
|
458
|
+
};
|
|
459
|
+
/** The module that promptfoo loads the function of each scorer assertion from (structured ADR 0013 in docs/adr/structured). */
|
|
408
460
|
export class AssertionModule<Name extends string> {
|
|
409
|
-
constructor(
|
|
410
|
-
url: string,
|
|
411
|
-
assertions: Readonly<Record<Name, ScorerAssertion<Weighted, unknown>>>,
|
|
412
|
-
);
|
|
461
|
+
constructor(url: string, assertions: Readonly<Record<Name, ModuleAssertion>>);
|
|
413
462
|
/** The default export of the module: the value function of each name. */
|
|
414
|
-
readonly functions: Readonly<
|
|
415
|
-
Record<string, ScorerAssertion<Weighted, unknown>['value']>
|
|
416
|
-
>;
|
|
463
|
+
readonly functions: Readonly<Record<string, ModuleAssertion['value']>>;
|
|
417
464
|
assert(...names: Name[]): FileAssertion[];
|
|
418
465
|
}
|
|
419
466
|
|
|
420
|
-
// @zukhruf/evals/classification
|
|
467
|
+
// @zukhruf/evals/classification
|
|
421
468
|
|
|
422
|
-
/** A label:
|
|
469
|
+
/** A label: a number or a string, as scikit-learn reads a label. */
|
|
423
470
|
export type Label = number | string;
|
|
424
471
|
|
|
425
472
|
/** The expected labels of one test, and its weight. */
|
|
426
473
|
export class LabelCase implements Weighted {
|
|
427
|
-
constructor(expected: Label | readonly Label[]
|
|
474
|
+
constructor(options: { expected: Label | readonly Label[]; weight: number });
|
|
428
475
|
readonly expected: ReadonlySet<Label>;
|
|
429
476
|
readonly weight: number;
|
|
430
477
|
}
|
|
431
|
-
/** Reads `vars.expected` (a label or a list of labels) and `vars.weight
|
|
478
|
+
/** Reads `vars.expected` (a label or a list of labels) and `vars.weight` (1 when none). */
|
|
432
479
|
export class LabelVarsCase implements CaseReader<LabelCase> {}
|
|
433
|
-
/** Reads `config.expected
|
|
480
|
+
/** Reads `config.expected`, with the weight 1. */
|
|
434
481
|
export class LabelConfigCase implements CaseReader<LabelCase> {}
|
|
435
482
|
|
|
436
483
|
export class LabelComparison {
|
|
484
|
+
constructor(sides: {
|
|
485
|
+
stated: ReadonlySet<Label>;
|
|
486
|
+
expected: ReadonlySet<Label>;
|
|
487
|
+
});
|
|
437
488
|
readonly stated: ReadonlySet<Label>;
|
|
438
489
|
readonly expected: ReadonlySet<Label>;
|
|
490
|
+
/** The labels of both sides, in the order of numpy's unique(). */
|
|
491
|
+
labels(): Label[];
|
|
492
|
+
/** The comparison of one class. */
|
|
493
|
+
only(label: Label): LabelComparison;
|
|
439
494
|
}
|
|
440
|
-
/** Reads the stated labels
|
|
441
|
-
export class LabelComparer implements Comparer<LabelCase, LabelComparison> {
|
|
442
|
-
constructor(options?: { normalize?: (label: string) => string });
|
|
443
|
-
}
|
|
495
|
+
/** Reads the stated labels: a label, a list of labels, a JSON array of labels, a number, or one string label. */
|
|
496
|
+
export class LabelComparer implements Comparer<LabelCase, LabelComparison> {}
|
|
444
497
|
|
|
445
498
|
export class ClassMatches implements Matches<LabelComparison> {
|
|
446
499
|
constructor(label: Label);
|
|
@@ -451,12 +504,20 @@ export class LabelsMatch extends Gate<LabelComparison> {}
|
|
|
451
504
|
export class PerClass extends Grouped<LabelComparison> {
|
|
452
505
|
constructor(
|
|
453
506
|
scorer: Scorer<LabelComparison>,
|
|
454
|
-
options
|
|
507
|
+
options?: { labels?: readonly Label[] },
|
|
455
508
|
);
|
|
456
509
|
}
|
|
457
510
|
|
|
511
|
+
/** The classes that an average reads. */
|
|
512
|
+
export type Classes = {
|
|
513
|
+
readonly selected: ReadonlyMap<Label, Counts>;
|
|
514
|
+
readonly present: ReadonlyMap<Label, Counts>;
|
|
515
|
+
readonly multilabel: boolean;
|
|
516
|
+
};
|
|
458
517
|
/** How the scores of the classes become one number (Strategy). */
|
|
459
|
-
export interface Average {
|
|
518
|
+
export interface Average {
|
|
519
|
+
of(classes: Classes, score: (counts: Counts) => number): number;
|
|
520
|
+
}
|
|
460
521
|
export class MicroAverage implements Average {}
|
|
461
522
|
export class MacroAverage implements Average {}
|
|
462
523
|
export class WeightedAverage implements Average {}
|
|
@@ -464,33 +525,33 @@ export class BinaryAverage implements Average {
|
|
|
464
525
|
constructor(positiveLabel: Label);
|
|
465
526
|
}
|
|
466
527
|
|
|
528
|
+
type ScoreOptions = {
|
|
529
|
+
average: Average;
|
|
530
|
+
labels?: readonly Label[];
|
|
531
|
+
/** 0 (the default), 1 or NaN. */
|
|
532
|
+
zeroDivision?: number;
|
|
533
|
+
};
|
|
467
534
|
/** The counts of each class over many answers: true positives, predicted and true. */
|
|
468
535
|
export class ClassCounts {
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
expected: readonly
|
|
472
|
-
|
|
473
|
-
|
|
536
|
+
constructor(options: { multilabel: boolean });
|
|
537
|
+
static of(answers: {
|
|
538
|
+
expected: readonly Label[];
|
|
539
|
+
stated: readonly Label[];
|
|
540
|
+
weights?: readonly number[];
|
|
541
|
+
}): ClassCounts;
|
|
542
|
+
static ofIndicators(answers: {
|
|
543
|
+
expected: readonly (readonly number[])[];
|
|
544
|
+
stated: readonly (readonly number[])[];
|
|
545
|
+
weights?: readonly number[];
|
|
546
|
+
}): ClassCounts;
|
|
547
|
+
add(comparison: LabelComparison, weight: number): void;
|
|
474
548
|
labels(): Label[];
|
|
475
549
|
accuracy(): number;
|
|
476
|
-
precision(options:
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
labels?: readonly Label[];
|
|
480
|
-
}): number;
|
|
481
|
-
recall(options: {
|
|
482
|
-
average: Average;
|
|
483
|
-
zeroDivision?: number;
|
|
484
|
-
labels?: readonly Label[];
|
|
485
|
-
}): number;
|
|
486
|
-
fBeta(options: {
|
|
487
|
-
beta: number;
|
|
488
|
-
average: Average;
|
|
489
|
-
zeroDivision?: number;
|
|
490
|
-
labels?: readonly Label[];
|
|
491
|
-
form?: FMeasureForm;
|
|
492
|
-
}): number;
|
|
550
|
+
precision(options: ScoreOptions): number;
|
|
551
|
+
recall(options: ScoreOptions): number;
|
|
552
|
+
fBeta(options: ScoreOptions & { beta: number; form?: FMeasureForm }): number;
|
|
493
553
|
support(): ReadonlyMap<Label, number>;
|
|
554
|
+
/** Throws for multilabel counts. */
|
|
494
555
|
confusionMatrix(): ConfusionMatrix;
|
|
495
556
|
}
|
|
496
557
|
export class ConfusionMatrix {
|
|
@@ -499,12 +560,24 @@ export class ConfusionMatrix {
|
|
|
499
560
|
rows(): number[][];
|
|
500
561
|
}
|
|
501
562
|
|
|
502
|
-
/** The stated probability of each class of one answer. */
|
|
563
|
+
/** The stated probability of each class of one answer, and its expected label. */
|
|
503
564
|
export class ProbabilityComparison {
|
|
504
565
|
readonly probabilities: ReadonlyMap<Label, number>;
|
|
505
|
-
readonly expected:
|
|
566
|
+
readonly expected: Label;
|
|
567
|
+
}
|
|
568
|
+
/** Reads the probability of each label from a Map, an object or its JSON text. */
|
|
569
|
+
export class ProbabilityComparer implements Comparer<
|
|
570
|
+
LabelCase,
|
|
571
|
+
ProbabilityComparison
|
|
572
|
+
> {}
|
|
573
|
+
export class BrierScore implements Scorer<ProbabilityComparison> {
|
|
574
|
+
constructor(options?: { positiveLabel?: Label });
|
|
575
|
+
/** The score of a whole run, with numpy's pairwise sum. */
|
|
576
|
+
mean(
|
|
577
|
+
comparisons: readonly ProbabilityComparison[],
|
|
578
|
+
weights?: readonly number[],
|
|
579
|
+
): number;
|
|
506
580
|
}
|
|
507
|
-
export class BrierScore implements Scorer<ProbabilityComparison> {}
|
|
508
581
|
```
|
|
509
582
|
|
|
510
583
|
## Who uses each part
|
|
@@ -520,7 +593,7 @@ The consumers below come from the search of 2026-11-06 (eight evals in four repo
|
|
|
520
593
|
| relevance-filtering eval | `./classification` (`BinaryAverage`, F0.5 with `CountsForm`, `BrierScore`) | mutex #3891 |
|
|
521
594
|
| event-detection eval | nothing yet: its MUC, B³, CEAF-e and BLANC wait for `./clustering` | — |
|
|
522
595
|
|
|
523
|
-
The
|
|
596
|
+
The package was built in two items: mutex #3888 (the core, `./promptfoo` and the move of `./structured`) and mutex #3889 (the generic core, the F-measure form and `./classification`).
|
|
524
597
|
|
|
525
598
|
## Documentation
|
|
526
599
|
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { Counts } from './counts.ts';
|
|
2
|
+
export interface FMeasureForm {
|
|
3
|
+
of(counts: Counts, beta: number): number;
|
|
4
|
+
}
|
|
5
|
+
export declare class CountsForm implements FMeasureForm {
|
|
6
|
+
of({ hits, stated, expected }: Counts, beta: number): number;
|
|
7
|
+
}
|
|
8
|
+
export declare class PrecisionRecallForm implements FMeasureForm {
|
|
9
|
+
of({ hits, stated, expected }: Counts, beta: number): number;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=f-measure-form.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f-measure-form.d.ts","sourceRoot":"","sources":["../../src/aggregation/f-measure-form.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAK1C,MAAM,WAAW,YAAY;IAC3B,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CAC1C;AAMD,qBAAa,UAAW,YAAW,YAAY;IAC7C,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;CAI7D;AAKD,qBAAa,mBAAoB,YAAW,YAAY;IACtD,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;CAO7D"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { Ratio } from "./ratio.js";
|
|
2
|
+
// (1 + β²)·hits / (β²·expected + stated), in the order of the operations of
|
|
3
|
+
// scikit-learn 1.9.1's fbeta_score (sklearn/metrics/_classification.py). It
|
|
4
|
+
// is the part of an F-measure ratio divided by its whole, and 0 when the
|
|
5
|
+
// whole is 0.
|
|
6
|
+
export class CountsForm {
|
|
7
|
+
of({ hits, stated, expected }, beta) {
|
|
8
|
+
const beta2 = beta ** 2;
|
|
9
|
+
return new Ratio((1 + beta2) * hits, beta2 * expected + stated).value;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
// (1 + β²)·P·R / (β²·P + R), precision and recall first (van Rijsbergen
|
|
13
|
+
// 1979), and 0 when there are no hits. With β = 1 it is 2·p·r/(p + r), the
|
|
14
|
+
// arithmetic of SOB's evaluate.py (f1_from_counts, token_f1).
|
|
15
|
+
export class PrecisionRecallForm {
|
|
16
|
+
of({ hits, stated, expected }, beta) {
|
|
17
|
+
if (hits === 0)
|
|
18
|
+
return 0;
|
|
19
|
+
const beta2 = beta ** 2;
|
|
20
|
+
const precision = hits / stated;
|
|
21
|
+
const recall = hits / expected;
|
|
22
|
+
return ((1 + beta2) * precision * recall) / (beta2 * precision + recall);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=f-measure-form.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f-measure-form.js","sourceRoot":"","sources":["../../src/aggregation/f-measure-form.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAQnC,4EAA4E;AAC5E,4EAA4E;AAC5E,yEAAyE;AACzE,cAAc;AACd,MAAM,OAAO,UAAU;IACrB,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAU,EAAE,IAAY;QACjD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC;QACxB,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC;IACxE,CAAC;CACF;AAED,wEAAwE;AACxE,2EAA2E;AAC3E,8DAA8D;AAC9D,MAAM,OAAO,mBAAmB;IAC9B,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAU,EAAE,IAAY;QACjD,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC;QACxB,MAAM,SAAS,GAAG,IAAI,GAAG,MAAM,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,GAAG,QAAQ,CAAC;QAC/B,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,GAAG,MAAM,CAAC,CAAC;IAC3E,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Counts } from './counts.ts';
|
|
2
|
+
import type { FMeasureForm } from './f-measure-form.ts';
|
|
3
|
+
import { Ratio } from './ratio.ts';
|
|
4
|
+
export declare class FMeasureRatio extends Ratio {
|
|
5
|
+
#private;
|
|
6
|
+
constructor(counts: Counts, { beta, form }: {
|
|
7
|
+
readonly beta: number;
|
|
8
|
+
readonly form: FMeasureForm;
|
|
9
|
+
});
|
|
10
|
+
get value(): number;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=f-measure-ratio.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f-measure-ratio.d.ts","sourceRoot":"","sources":["../../src/aggregation/f-measure-ratio.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMnC,qBAAa,aAAc,SAAQ,KAAK;;gBAMpC,MAAM,EAAE,MAAM,EACd,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAA;KAAE;IASxE,IAAa,KAAK,IAAI,MAAM,CAE3B;CACF"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Ratio } from "./ratio.js";
|
|
2
|
+
// The ratio of an F-measure: its part is (1 + β²)·hits and its whole is
|
|
3
|
+
// β²·expected + stated, so many of them add up to the micro F-measure. Its
|
|
4
|
+
// value is the value of its form, which can differ from part / whole in the
|
|
5
|
+
// last bit (ADR 0003).
|
|
6
|
+
export class FMeasureRatio extends Ratio {
|
|
7
|
+
#counts;
|
|
8
|
+
#beta;
|
|
9
|
+
#form;
|
|
10
|
+
constructor(counts, { beta, form }) {
|
|
11
|
+
const beta2 = beta ** 2;
|
|
12
|
+
super((1 + beta2) * counts.hits, beta2 * counts.expected + counts.stated);
|
|
13
|
+
this.#counts = counts;
|
|
14
|
+
this.#beta = beta;
|
|
15
|
+
this.#form = form;
|
|
16
|
+
}
|
|
17
|
+
get value() {
|
|
18
|
+
return this.#form.of(this.#counts, this.#beta);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=f-measure-ratio.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"f-measure-ratio.js","sourceRoot":"","sources":["../../src/aggregation/f-measure-ratio.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,wEAAwE;AACxE,2EAA2E;AAC3E,4EAA4E;AAC5E,uBAAuB;AACvB,MAAM,OAAO,aAAc,SAAQ,KAAK;IAC7B,OAAO,CAAS;IAChB,KAAK,CAAS;IACd,KAAK,CAAe;IAE7B,YACE,MAAc,EACd,EAAE,IAAI,EAAE,IAAI,EAA0D;QAEtE,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC;QACxB,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1E,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,IAAa,KAAK;QAChB,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACjD,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ratio-of-sums.d.ts","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMnC,qBAAa,WAAW;;
|
|
1
|
+
{"version":3,"file":"ratio-of-sums.d.ts","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMnC,qBAAa,WAAW;;IAKtB,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,WAAW;IAM/C,GAAG,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;IAKvB,IAAI,KAAK,IAAI,KAAK,CAEjB;IAED,IAAI,KAAK,IAAI,MAAM,CAElB;CACF"}
|
|
@@ -7,6 +7,13 @@ import { Ratio } from "./ratio.js";
|
|
|
7
7
|
export class RatioOfSums {
|
|
8
8
|
#parts = new NeumaierSum();
|
|
9
9
|
#wholes = new NeumaierSum();
|
|
10
|
+
// The ratio of sums of the ratios, in their order.
|
|
11
|
+
static of(ratios) {
|
|
12
|
+
const sum = new RatioOfSums();
|
|
13
|
+
for (const ratio of ratios)
|
|
14
|
+
sum.add(ratio);
|
|
15
|
+
return sum;
|
|
16
|
+
}
|
|
10
17
|
add(ratio) {
|
|
11
18
|
this.#parts.add(ratio.part);
|
|
12
19
|
this.#wholes.add(ratio.whole);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ratio-of-sums.js","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,0EAA0E;AAC1E,sEAAsE;AACtE,2EAA2E;AAC3E,sCAAsC;AACtC,MAAM,OAAO,WAAW;IACb,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;IAC3B,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAErC,GAAG,CAAC,KAAY;QACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;IAC1B,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"ratio-of-sums.js","sourceRoot":"","sources":["../../src/aggregation/ratio-of-sums.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,0EAA0E;AAC1E,sEAAsE;AACtE,2EAA2E;AAC3E,sCAAsC;AACtC,MAAM,OAAO,WAAW;IACb,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;IAC3B,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAErC,mDAAmD;IACnD,MAAM,CAAC,EAAE,CAAC,MAAuB;QAC/B,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,KAAK,MAAM,KAAK,IAAI,MAAM;YAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3C,OAAO,GAAG,CAAC;IACb,CAAC;IAED,GAAG,CAAC,KAAY;QACd,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;IAC1B,CAAC;CACF"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Counts } from '../aggregation/counts.ts';
|
|
2
|
+
import type { Matches } from '../scorers/matches.ts';
|
|
3
|
+
import type { LabelComparison } from './label-comparison.ts';
|
|
4
|
+
export declare class AllClassMatches implements Matches<LabelComparison> {
|
|
5
|
+
counts({ stated, expected }: LabelComparison): Counts;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=all-class-matches.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"all-class-matches.d.ts","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAK7D,qBAAa,eAAgB,YAAW,OAAO,CAAC,eAAe,CAAC;IAC9D,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,eAAe,GAAG,MAAM;CAOtD"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Counts } from "../aggregation/counts.js";
|
|
2
|
+
// The counts of every class in one answer: the hits are the labels that both
|
|
3
|
+
// sides have, the stated and the expected are the labels of each side.
|
|
4
|
+
// Pooled over the answers, they are scikit-learn's micro counts.
|
|
5
|
+
export class AllClassMatches {
|
|
6
|
+
counts({ stated, expected }) {
|
|
7
|
+
return new Counts({
|
|
8
|
+
hits: stated.intersection(expected).size,
|
|
9
|
+
stated: stated.size,
|
|
10
|
+
expected: expected.size,
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=all-class-matches.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"all-class-matches.js","sourceRoot":"","sources":["../../src/classification/all-class-matches.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAIlD,6EAA6E;AAC7E,uEAAuE;AACvE,iEAAiE;AACjE,MAAM,OAAO,eAAe;IAC1B,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAmB;QAC1C,OAAO,IAAI,MAAM,CAAC;YAChB,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI;YACxC,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,QAAQ,EAAE,QAAQ,CAAC,IAAI;SACxB,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { Counts } from '../aggregation/counts.ts';
|
|
2
|
+
import type { Label } from './label.ts';
|
|
3
|
+
export type Classes = {
|
|
4
|
+
readonly selected: ReadonlyMap<Label, Counts>;
|
|
5
|
+
readonly present: ReadonlyMap<Label, Counts>;
|
|
6
|
+
readonly multilabel: boolean;
|
|
7
|
+
};
|
|
8
|
+
export interface Average {
|
|
9
|
+
of(classes: Classes, score: (counts: Counts) => number): number;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=average.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"average.d.ts","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAKxC,MAAM,MAAM,OAAO,GAAG;IACpB,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC9C,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;CAC9B,CAAC;AAKF,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,CAAC;CACjE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"average.js","sourceRoot":"","sources":["../../src/classification/average.ts"],"names":[],"mappings":""}
|