@arizeai/phoenix-evals 1.0.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -0
- package/dist/esm/code/classificationMetrics.d.ts +121 -0
- package/dist/esm/code/classificationMetrics.d.ts.map +1 -0
- package/dist/esm/code/classificationMetrics.js +269 -0
- package/dist/esm/code/classificationMetrics.js.map +1 -0
- package/dist/esm/code/createClassificationMetricEvaluator.d.ts +16 -0
- package/dist/esm/code/createClassificationMetricEvaluator.d.ts.map +1 -0
- package/dist/esm/code/createClassificationMetricEvaluator.js +23 -0
- package/dist/esm/code/createClassificationMetricEvaluator.js.map +1 -0
- package/dist/esm/code/createF1Evaluator.d.ts +23 -0
- package/dist/esm/code/createF1Evaluator.d.ts.map +1 -0
- package/dist/esm/code/createF1Evaluator.js +23 -0
- package/dist/esm/code/createF1Evaluator.js.map +1 -0
- package/dist/esm/code/createFBetaEvaluator.d.ts +23 -0
- package/dist/esm/code/createFBetaEvaluator.d.ts.map +1 -0
- package/dist/esm/code/createFBetaEvaluator.js +27 -0
- package/dist/esm/code/createFBetaEvaluator.js.map +1 -0
- package/dist/esm/code/createPrecisionEvaluator.d.ts +22 -0
- package/dist/esm/code/createPrecisionEvaluator.d.ts.map +1 -0
- package/dist/esm/code/createPrecisionEvaluator.js +25 -0
- package/dist/esm/code/createPrecisionEvaluator.js.map +1 -0
- package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
- package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
- package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js +44 -0
- package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
- package/dist/esm/code/createRecallEvaluator.d.ts +22 -0
- package/dist/esm/code/createRecallEvaluator.d.ts.map +1 -0
- package/dist/esm/code/createRecallEvaluator.js +25 -0
- package/dist/esm/code/createRecallEvaluator.js.map +1 -0
- package/dist/esm/code/index.d.ts +7 -0
- package/dist/esm/code/index.d.ts.map +1 -0
- package/dist/esm/code/index.js +7 -0
- package/dist/esm/code/index.js.map +1 -0
- package/dist/esm/index.d.ts +1 -0
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +1 -0
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/src/code/classificationMetrics.d.ts +121 -0
- package/dist/src/code/classificationMetrics.d.ts.map +1 -0
- package/dist/src/code/classificationMetrics.js +274 -0
- package/dist/src/code/classificationMetrics.js.map +1 -0
- package/dist/src/code/createClassificationMetricEvaluator.d.ts +16 -0
- package/dist/src/code/createClassificationMetricEvaluator.d.ts.map +1 -0
- package/dist/src/code/createClassificationMetricEvaluator.js +26 -0
- package/dist/src/code/createClassificationMetricEvaluator.js.map +1 -0
- package/dist/src/code/createF1Evaluator.d.ts +23 -0
- package/dist/src/code/createF1Evaluator.d.ts.map +1 -0
- package/dist/src/code/createF1Evaluator.js +26 -0
- package/dist/src/code/createF1Evaluator.js.map +1 -0
- package/dist/src/code/createFBetaEvaluator.d.ts +23 -0
- package/dist/src/code/createFBetaEvaluator.d.ts.map +1 -0
- package/dist/src/code/createFBetaEvaluator.js +30 -0
- package/dist/src/code/createFBetaEvaluator.js.map +1 -0
- package/dist/src/code/createPrecisionEvaluator.d.ts +22 -0
- package/dist/src/code/createPrecisionEvaluator.d.ts.map +1 -0
- package/dist/src/code/createPrecisionEvaluator.js +28 -0
- package/dist/src/code/createPrecisionEvaluator.js.map +1 -0
- package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
- package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
- package/dist/src/code/createPrecisionRecallFScoreEvaluators.js +47 -0
- package/dist/src/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
- package/dist/src/code/createRecallEvaluator.d.ts +22 -0
- package/dist/src/code/createRecallEvaluator.d.ts.map +1 -0
- package/dist/src/code/createRecallEvaluator.js +28 -0
- package/dist/src/code/createRecallEvaluator.js.map +1 -0
- package/dist/src/code/index.d.ts +7 -0
- package/dist/src/code/index.d.ts.map +1 -0
- package/dist/src/code/index.js +23 -0
- package/dist/src/code/index.js.map +1 -0
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -0
- package/dist/src/index.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/docs/classification-metrics.mdx +103 -0
- package/docs/overview.mdx +4 -1
- package/package.json +24 -16
- package/src/code/classificationMetrics.ts +464 -0
- package/src/code/createClassificationMetricEvaluator.ts +49 -0
- package/src/code/createF1Evaluator.ts +32 -0
- package/src/code/createFBetaEvaluator.ts +41 -0
- package/src/code/createPrecisionEvaluator.ts +36 -0
- package/src/code/createPrecisionRecallFScoreEvaluators.ts +83 -0
- package/src/code/createRecallEvaluator.ts +36 -0
- package/src/code/index.ts +6 -0
- package/src/index.ts +1 -0
package/README.md
CHANGED
|
@@ -145,6 +145,44 @@ console.log(relevanceResult);
|
|
|
145
145
|
// Output: { label: "relevant", score: 1, explanation: "..." }
|
|
146
146
|
```
|
|
147
147
|
|
|
148
|
+
### Code Evaluators
|
|
149
|
+
|
|
150
|
+
The library also includes built-in, deterministic (non-LLM) code evaluators for common classification metrics: precision, recall, and F-beta (including F1). These are available from the `@arizeai/phoenix-evals/code` module and work over a batch of expected vs. predicted labels, supporting both binary classification (via `positiveLabel`) and multi-class classification (via `macro`/`micro`/`weighted` averaging).
|
|
151
|
+
|
|
152
|
+
- **Precision** — of everything predicted as a class, the fraction that was actually that class (`TP / (TP + FP)`). Lower precision means more false alarms.
|
|
153
|
+
- **Recall** — of everything that actually belongs to a class, the fraction the model found (`TP / (TP + FN)`). Lower recall means more misses.
|
|
154
|
+
- **F-beta** — the weighted harmonic mean of precision and recall. `beta = 1` (F1, the default) weights them equally; `beta > 1` weights recall more (use when missing a true positive is costlier, e.g. medical screening); `beta < 1` weights precision more (use when a false alarm is costlier, e.g. spam filtering).
|
|
155
|
+
|
|
156
|
+
```typescript
|
|
157
|
+
import {
|
|
158
|
+
createPrecisionEvaluator,
|
|
159
|
+
createRecallEvaluator,
|
|
160
|
+
createF1Evaluator,
|
|
161
|
+
createPrecisionRecallFScoreEvaluators,
|
|
162
|
+
} from "@arizeai/phoenix-evals/code";
|
|
163
|
+
|
|
164
|
+
const f1 = createF1Evaluator();
|
|
165
|
+
const result = await f1.evaluate({
|
|
166
|
+
expected: ["cat", "dog", "cat", "bird"],
|
|
167
|
+
output: ["cat", "cat", "cat", "bird"],
|
|
168
|
+
});
|
|
169
|
+
console.log(result);
|
|
170
|
+
// Output: { score: 0.6 }
|
|
171
|
+
|
|
172
|
+
// Or create matching precision/recall/F-score evaluators at once
|
|
173
|
+
const { precision, recall, fScore } = createPrecisionRecallFScoreEvaluators({
|
|
174
|
+
average: "weighted",
|
|
175
|
+
});
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Use `createFBetaEvaluator({ beta })` for F-scores other than F1 (e.g. `beta: 2` weights recall higher than precision). `createPrecisionRecallFScoreEvaluators` returns three separate evaluators (`{ precision, recall, fScore }`) sharing one options object — each yields a single score, matching the individual factories.
|
|
179
|
+
|
|
180
|
+
For multi-class data, `average` controls how per-class scores combine into one number: `"macro"` (default) weights every class equally — good for surfacing whether a rare class is being ignored; `"weighted"` weights each class by how often it occurs — good when overall performance matters more than parity across classes; `"micro"` pools every true/false positive and false negative across classes first — for single-label multi-class problems this equals overall accuracy.
|
|
181
|
+
|
|
182
|
+
See [`examples/classification_metrics_example.ts`](examples/classification_metrics_example.ts) for a runnable walkthrough covering binary classification, F-beta tradeoffs, and all three averaging strategies. For the underlying formulas, the precision/recall tradeoff, and citations, see the [Precision / Recall / F-Score docs](https://arize.com/docs/phoenix/evaluation/pre-built-metrics/precision-recall-fscore).
|
|
183
|
+
|
|
184
|
+
> **Note:** unlike the per-row LLM evaluators above, these classification-metric evaluators are batch/dataset-level: `expected`/`output` are the full sequence of labels across every example, not a single row's labels. Don't wire them directly into `runExperiment` as a per-row evaluator — instead, collect every row's expected/predicted label first, then call `.evaluate({ expected, output })` once over the full arrays.
|
|
185
|
+
|
|
148
186
|
### Data Mapping
|
|
149
187
|
|
|
150
188
|
When your data structure doesn't match what an evaluator expects, use `bindEvaluator` to map your fields to the evaluator's expected input format:
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A label produced by a classifier. Must be a primitive so it can be compared
|
|
3
|
+
* for equality and used as a lookup key.
|
|
4
|
+
*/
|
|
5
|
+
export type ClassificationLabel = string | number;
|
|
6
|
+
/**
|
|
7
|
+
* The strategy used to aggregate per-class precision/recall/F-score into a
|
|
8
|
+
* single number when there are more than two classes (or no `positiveLabel`
|
|
9
|
+
* is configured).
|
|
10
|
+
*
|
|
11
|
+
* - `"macro"`: unweighted mean across classes.
|
|
12
|
+
* - `"micro"`: pool true/false positives and false negatives across classes
|
|
13
|
+
* before computing the metric.
|
|
14
|
+
* - `"weighted"`: mean across classes, weighted by each class's support
|
|
15
|
+
* (number of true instances).
|
|
16
|
+
*/
|
|
17
|
+
export type AverageType = "macro" | "micro" | "weighted";
|
|
18
|
+
/**
|
|
19
|
+
* The example shape expected by the classification-metric evaluators.
|
|
20
|
+
*/
|
|
21
|
+
export interface ClassificationExample {
|
|
22
|
+
/** The ground-truth sequence of labels. */
|
|
23
|
+
expected: ClassificationLabel[];
|
|
24
|
+
/** The predicted sequence of labels, aligned by index with `expected`. */
|
|
25
|
+
output: ClassificationLabel[];
|
|
26
|
+
[key: string]: unknown;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Options shared by precision, recall, and F-score computations.
|
|
30
|
+
*/
|
|
31
|
+
export interface PrecisionRecallFScoreOptions {
|
|
32
|
+
/**
|
|
33
|
+
* Weight of recall relative to precision in the F-score. Must be > 0.
|
|
34
|
+
* @defaultValue 1
|
|
35
|
+
*/
|
|
36
|
+
beta?: number;
|
|
37
|
+
/**
|
|
38
|
+
* Aggregation strategy across classes. Ignored when `positiveLabel` is set
|
|
39
|
+
* (or auto-detected).
|
|
40
|
+
* @defaultValue "macro"
|
|
41
|
+
*/
|
|
42
|
+
average?: AverageType;
|
|
43
|
+
/**
|
|
44
|
+
* Value substituted for a metric when it is undefined (e.g. 0/0).
|
|
45
|
+
* @defaultValue 0
|
|
46
|
+
*/
|
|
47
|
+
zeroDivision?: number;
|
|
48
|
+
/**
|
|
49
|
+
* When set, compute binary precision/recall/F exclusively for this label
|
|
50
|
+
* (one-vs-rest). If not set, `average` is at its default `"macro"`, and
|
|
51
|
+
* the labels are the numeric set `{0, 1}`, the positive label defaults to
|
|
52
|
+
* `1`. Otherwise, multi-class averaging is used. The auto-detection is
|
|
53
|
+
* skipped whenever a non-default `average` is configured, so an explicit
|
|
54
|
+
* `average` is never silently overridden by the shape of the data.
|
|
55
|
+
*/
|
|
56
|
+
positiveLabel?: ClassificationLabel;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* The result of computing precision, recall, and F-score for a batch of
|
|
60
|
+
* predictions.
|
|
61
|
+
*/
|
|
62
|
+
export interface PrecisionRecallFScoreResult {
|
|
63
|
+
precision: number;
|
|
64
|
+
recall: number;
|
|
65
|
+
fScore: number;
|
|
66
|
+
beta: number;
|
|
67
|
+
average: AverageType;
|
|
68
|
+
/** All labels observed in `expected` and `output`, in first-seen order. */
|
|
69
|
+
labels: ClassificationLabel[];
|
|
70
|
+
/** The label treated as positive in one-vs-rest mode, or `null` if multi-class averaging was used. */
|
|
71
|
+
positiveLabel: ClassificationLabel | null;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Computes precision, recall, and F-beta score for a batch of expected vs.
|
|
75
|
+
* predicted labels.
|
|
76
|
+
*
|
|
77
|
+
* `expected`/`output` are the full sequence of labels across an entire
|
|
78
|
+
* dataset, not a single row — this and the evaluators built on it are
|
|
79
|
+
* dataset-level, unlike the package's per-row LLM evaluators. Call it once
|
|
80
|
+
* over every row's collected labels rather than wiring it into a per-row
|
|
81
|
+
* pipeline (e.g. `runExperiment`'s per-row evaluators).
|
|
82
|
+
*
|
|
83
|
+
* Supports both binary classification (via `positiveLabel`, or
|
|
84
|
+
* auto-detected when `average` is at its default `"macro"` and the labels
|
|
85
|
+
* are the numeric set `{0, 1}`) and multi-class classification (via the
|
|
86
|
+
* `average` strategy).
|
|
87
|
+
*
|
|
88
|
+
* @example Multi-class (macro average)
|
|
89
|
+
* ```typescript
|
|
90
|
+
* computePrecisionRecallFScore({
|
|
91
|
+
* expected: ["cat", "dog", "cat", "bird"],
|
|
92
|
+
* output: ["cat", "cat", "cat", "bird"],
|
|
93
|
+
* });
|
|
94
|
+
* // { precision: 5/9, recall: 2/3, fScore: 0.6, beta: 1, average: "macro", ... }
|
|
95
|
+
* ```
|
|
96
|
+
*
|
|
97
|
+
* @example Binary with an explicit positive label
|
|
98
|
+
* ```typescript
|
|
99
|
+
* computePrecisionRecallFScore(
|
|
100
|
+
* { expected: ["spam", "ham", "spam"], output: ["spam", "spam", "ham"] },
|
|
101
|
+
* { beta: 0.5, positiveLabel: "spam" }
|
|
102
|
+
* );
|
|
103
|
+
* ```
|
|
104
|
+
*/
|
|
105
|
+
export declare function computePrecisionRecallFScore({ expected, output }: Pick<ClassificationExample, "expected" | "output">, options?: PrecisionRecallFScoreOptions): PrecisionRecallFScoreResult;
|
|
106
|
+
/**
|
|
107
|
+
* Formats a beta value for use in metric names, e.g. `1` -> `"f1"`,
|
|
108
|
+
* `0.5` -> `"f0_5"`.
|
|
109
|
+
*/
|
|
110
|
+
export declare function formatBetaForMetricName(beta: number): string;
|
|
111
|
+
/**
|
|
112
|
+
* The suffix appended to a metric name to reflect the aggregation strategy,
|
|
113
|
+
* e.g. `"precision"` vs. `"precision_micro"`. No suffix is used when
|
|
114
|
+
* `positiveLabel` is explicitly configured, since `average` is not
|
|
115
|
+
* applicable in that one-vs-rest binary mode. This mirrors
|
|
116
|
+
* `resolvePositiveLabel`'s auto-detection rule (only under the default
|
|
117
|
+
* `"macro"` average) so a constructed evaluator's static name always
|
|
118
|
+
* matches what `computePrecisionRecallFScore` actually computes.
|
|
119
|
+
*/
|
|
120
|
+
export declare function getAverageMetricNameSuffix({ average, positiveLabel, }: PrecisionRecallFScoreOptions): string;
|
|
121
|
+
//# sourceMappingURL=classificationMetrics.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"classificationMetrics.d.ts","sourceRoot":"","sources":["../../../src/code/classificationMetrics.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,GAAG,MAAM,CAAC;AAElD;;;;;;;;;;GAUG;AACH,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,2CAA2C;IAC3C,QAAQ,EAAE,mBAAmB,EAAE,CAAC;IAChC,0EAA0E;IAC1E,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAC9B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,mBAAmB,CAAC;CACrC;AAED;;;GAGG;AACH,MAAM,WAAW,2BAA2B;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,CAAC;IACrB,2EAA2E;IAC3E,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAC9B,sGAAsG;IACtG,aAAa,EAAE,mBAAmB,GAAG,IAAI,CAAC;CAC3C;AAiQD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,wBAAgB,4BAA4B,CAC1C,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,qBAAqB,EAAE,UAAU,GAAG,QAAQ,CAAC,EACxE,OAAO,GAAE,4BAAiC,GACzC,2BAA2B,CA+D7B;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQ5D;AAED;;;;;;;;GAQG;AACH,wBAAgB,0BAA0B,CAAC,EACzC,OAAiB,EACjB,aAAa,GACd,EAAE,4BAA4B,GAAG,MAAM,CAKvC"}
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
function validateBeta(beta) {
|
|
2
|
+
if (!(beta > 0)) {
|
|
3
|
+
throw new Error("beta must be > 0");
|
|
4
|
+
}
|
|
5
|
+
}
|
|
6
|
+
function validateAverage(average) {
|
|
7
|
+
if (average !== "macro" && average !== "micro" && average !== "weighted") {
|
|
8
|
+
throw new Error('average must be one of "macro", "micro", or "weighted"');
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
function validateLabelSequences(expected, output) {
|
|
12
|
+
if (!Array.isArray(expected) || !Array.isArray(output)) {
|
|
13
|
+
throw new Error("expected and output must be arrays of labels");
|
|
14
|
+
}
|
|
15
|
+
if (expected.length !== output.length) {
|
|
16
|
+
throw new Error(`expected and output must have the same length. Got ${expected.length} and ${output.length}`);
|
|
17
|
+
}
|
|
18
|
+
if (expected.length === 0) {
|
|
19
|
+
throw new Error("expected and output must be non-empty");
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function safeDivide(numerator, denominator, zeroDivision) {
|
|
23
|
+
if (denominator === 0) {
|
|
24
|
+
return zeroDivision;
|
|
25
|
+
}
|
|
26
|
+
return numerator / denominator;
|
|
27
|
+
}
|
|
28
|
+
function computeFScore(precision, recall, beta, zeroDivision) {
|
|
29
|
+
if (precision === 0 && recall === 0) {
|
|
30
|
+
return 0;
|
|
31
|
+
}
|
|
32
|
+
const betaSquared = beta * beta;
|
|
33
|
+
const numerator = (1 + betaSquared) * precision * recall;
|
|
34
|
+
const denominator = betaSquared * precision + recall;
|
|
35
|
+
return safeDivide(numerator, denominator, zeroDivision);
|
|
36
|
+
}
|
|
37
|
+
/** Collects all unique labels, ordered by first appearance in `expected` then `output`. */
|
|
38
|
+
function collectLabels(expected, output) {
|
|
39
|
+
const seen = new Set();
|
|
40
|
+
const ordered = [];
|
|
41
|
+
for (const label of [...expected, ...output]) {
|
|
42
|
+
if (!seen.has(label)) {
|
|
43
|
+
seen.add(label);
|
|
44
|
+
ordered.push(label);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return ordered;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Compares two labels the same way `Map`/`Set` key lookups do (SameValueZero),
|
|
51
|
+
* so a `NaN` label matches another `NaN` label instead of always mismatching
|
|
52
|
+
* under strict `===`.
|
|
53
|
+
*/
|
|
54
|
+
function labelsAreEqual(a, b) {
|
|
55
|
+
return (a === b ||
|
|
56
|
+
(typeof a === "number" &&
|
|
57
|
+
typeof b === "number" &&
|
|
58
|
+
Number.isNaN(a) &&
|
|
59
|
+
Number.isNaN(b)));
|
|
60
|
+
}
|
|
61
|
+
function computeClassCounts(expected, output, labels) {
|
|
62
|
+
const countsByLabel = new Map();
|
|
63
|
+
for (const label of labels) {
|
|
64
|
+
countsByLabel.set(label, {
|
|
65
|
+
truePositive: 0,
|
|
66
|
+
falsePositive: 0,
|
|
67
|
+
falseNegative: 0,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
for (const [index, expectedLabel] of expected.entries()) {
|
|
71
|
+
// `output` is validated to have the same length as `expected`.
|
|
72
|
+
const outputLabel = output[index];
|
|
73
|
+
// `labels` is derived from these same `expected`/`output` arrays, so both
|
|
74
|
+
// labels are always pre-populated keys in `countsByLabel`.
|
|
75
|
+
if (labelsAreEqual(expectedLabel, outputLabel)) {
|
|
76
|
+
const counts = countsByLabel.get(expectedLabel);
|
|
77
|
+
counts.truePositive += 1;
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
const predictedCounts = countsByLabel.get(outputLabel);
|
|
81
|
+
predictedCounts.falsePositive += 1;
|
|
82
|
+
const expectedCounts = countsByLabel.get(expectedLabel);
|
|
83
|
+
expectedCounts.falseNegative += 1;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return countsByLabel;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Resolves the one-vs-rest positive label, if any. Auto-detection of a
|
|
90
|
+
* numeric `{0, 1}` label set only applies under the default `"macro"`
|
|
91
|
+
* average, so an explicitly configured `"micro"`/`"weighted"` average is
|
|
92
|
+
* never silently replaced by binary one-vs-rest scoring.
|
|
93
|
+
*/
|
|
94
|
+
function resolvePositiveLabel(configuredPositiveLabel, labels, average) {
|
|
95
|
+
if (configuredPositiveLabel !== undefined) {
|
|
96
|
+
return configuredPositiveLabel;
|
|
97
|
+
}
|
|
98
|
+
if (average !== "macro") {
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
const uniqueLabels = new Set(labels);
|
|
102
|
+
if (uniqueLabels.size === 2 && uniqueLabels.has(0) && uniqueLabels.has(1)) {
|
|
103
|
+
return 1;
|
|
104
|
+
}
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Aggregates per-class precision/recall/F-score across all classes.
|
|
109
|
+
*
|
|
110
|
+
* Mirrors scikit-learn's `average` semantics: for `"macro"`/`"weighted"` the
|
|
111
|
+
* F-score is computed per class and then averaged (not derived from the
|
|
112
|
+
* aggregated precision/recall). For `"micro"` the per-class confusion counts
|
|
113
|
+
* are pooled first, so the F-score is computed from the pooled
|
|
114
|
+
* precision/recall.
|
|
115
|
+
*/
|
|
116
|
+
function aggregatePrecisionRecall(countsByLabel, average, beta, zeroDivision) {
|
|
117
|
+
if (average === "micro") {
|
|
118
|
+
let truePositive = 0;
|
|
119
|
+
let falsePositive = 0;
|
|
120
|
+
let falseNegative = 0;
|
|
121
|
+
for (const counts of countsByLabel.values()) {
|
|
122
|
+
truePositive += counts.truePositive;
|
|
123
|
+
falsePositive += counts.falsePositive;
|
|
124
|
+
falseNegative += counts.falseNegative;
|
|
125
|
+
}
|
|
126
|
+
const precision = safeDivide(truePositive, truePositive + falsePositive, zeroDivision);
|
|
127
|
+
const recall = safeDivide(truePositive, truePositive + falseNegative, zeroDivision);
|
|
128
|
+
return {
|
|
129
|
+
precision,
|
|
130
|
+
recall,
|
|
131
|
+
fScore: computeFScore(precision, recall, beta, zeroDivision),
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
const perClassMetrics = Array.from(countsByLabel.values()).map((counts) => {
|
|
135
|
+
const precision = safeDivide(counts.truePositive, counts.truePositive + counts.falsePositive, zeroDivision);
|
|
136
|
+
const recall = safeDivide(counts.truePositive, counts.truePositive + counts.falseNegative, zeroDivision);
|
|
137
|
+
return {
|
|
138
|
+
precision,
|
|
139
|
+
recall,
|
|
140
|
+
fScore: computeFScore(precision, recall, beta, zeroDivision),
|
|
141
|
+
support: counts.truePositive + counts.falseNegative,
|
|
142
|
+
};
|
|
143
|
+
});
|
|
144
|
+
if (average === "macro") {
|
|
145
|
+
return {
|
|
146
|
+
precision: mean(perClassMetrics.map((m) => m.precision)),
|
|
147
|
+
recall: mean(perClassMetrics.map((m) => m.recall)),
|
|
148
|
+
fScore: mean(perClassMetrics.map((m) => m.fScore)),
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
// weighted
|
|
152
|
+
const totalSupport = perClassMetrics.reduce((sum, m) => sum + m.support, 0);
|
|
153
|
+
if (totalSupport === 0) {
|
|
154
|
+
return {
|
|
155
|
+
precision: zeroDivision,
|
|
156
|
+
recall: zeroDivision,
|
|
157
|
+
fScore: zeroDivision,
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
return {
|
|
161
|
+
precision: weightedMean(perClassMetrics, "precision", totalSupport),
|
|
162
|
+
recall: weightedMean(perClassMetrics, "recall", totalSupport),
|
|
163
|
+
fScore: weightedMean(perClassMetrics, "fScore", totalSupport),
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
function mean(values) {
|
|
167
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
168
|
+
}
|
|
169
|
+
function weightedMean(perClassMetrics, field, totalSupport) {
|
|
170
|
+
const weightedSum = perClassMetrics.reduce((sum, metrics) => sum + metrics[field] * metrics.support, 0);
|
|
171
|
+
return weightedSum / totalSupport;
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Computes precision, recall, and F-beta score for a batch of expected vs.
|
|
175
|
+
* predicted labels.
|
|
176
|
+
*
|
|
177
|
+
* `expected`/`output` are the full sequence of labels across an entire
|
|
178
|
+
* dataset, not a single row — this and the evaluators built on it are
|
|
179
|
+
* dataset-level, unlike the package's per-row LLM evaluators. Call it once
|
|
180
|
+
* over every row's collected labels rather than wiring it into a per-row
|
|
181
|
+
* pipeline (e.g. `runExperiment`'s per-row evaluators).
|
|
182
|
+
*
|
|
183
|
+
* Supports both binary classification (via `positiveLabel`, or
|
|
184
|
+
* auto-detected when `average` is at its default `"macro"` and the labels
|
|
185
|
+
* are the numeric set `{0, 1}`) and multi-class classification (via the
|
|
186
|
+
* `average` strategy).
|
|
187
|
+
*
|
|
188
|
+
* @example Multi-class (macro average)
|
|
189
|
+
* ```typescript
|
|
190
|
+
* computePrecisionRecallFScore({
|
|
191
|
+
* expected: ["cat", "dog", "cat", "bird"],
|
|
192
|
+
* output: ["cat", "cat", "cat", "bird"],
|
|
193
|
+
* });
|
|
194
|
+
* // { precision: 5/9, recall: 2/3, fScore: 0.6, beta: 1, average: "macro", ... }
|
|
195
|
+
* ```
|
|
196
|
+
*
|
|
197
|
+
* @example Binary with an explicit positive label
|
|
198
|
+
* ```typescript
|
|
199
|
+
* computePrecisionRecallFScore(
|
|
200
|
+
* { expected: ["spam", "ham", "spam"], output: ["spam", "spam", "ham"] },
|
|
201
|
+
* { beta: 0.5, positiveLabel: "spam" }
|
|
202
|
+
* );
|
|
203
|
+
* ```
|
|
204
|
+
*/
|
|
205
|
+
export function computePrecisionRecallFScore({ expected, output }, options = {}) {
|
|
206
|
+
const { beta = 1, average = "macro", zeroDivision = 0, positiveLabel, } = options;
|
|
207
|
+
validateBeta(beta);
|
|
208
|
+
validateAverage(average);
|
|
209
|
+
validateLabelSequences(expected, output);
|
|
210
|
+
const labels = collectLabels(expected, output);
|
|
211
|
+
const countsByLabel = computeClassCounts(expected, output, labels);
|
|
212
|
+
const resolvedPositiveLabel = resolvePositiveLabel(positiveLabel, labels, average);
|
|
213
|
+
if (resolvedPositiveLabel !== null) {
|
|
214
|
+
const classCounts = countsByLabel.get(resolvedPositiveLabel);
|
|
215
|
+
if (!classCounts) {
|
|
216
|
+
throw new Error(`positiveLabel ${JSON.stringify(resolvedPositiveLabel)} not present in labels ${JSON.stringify(labels)}`);
|
|
217
|
+
}
|
|
218
|
+
const precision = safeDivide(classCounts.truePositive, classCounts.truePositive + classCounts.falsePositive, zeroDivision);
|
|
219
|
+
const recall = safeDivide(classCounts.truePositive, classCounts.truePositive + classCounts.falseNegative, zeroDivision);
|
|
220
|
+
return {
|
|
221
|
+
precision,
|
|
222
|
+
recall,
|
|
223
|
+
fScore: computeFScore(precision, recall, beta, zeroDivision),
|
|
224
|
+
beta,
|
|
225
|
+
average,
|
|
226
|
+
labels,
|
|
227
|
+
positiveLabel: resolvedPositiveLabel,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
const { precision, recall, fScore } = aggregatePrecisionRecall(countsByLabel, average, beta, zeroDivision);
|
|
231
|
+
return {
|
|
232
|
+
precision,
|
|
233
|
+
recall,
|
|
234
|
+
fScore,
|
|
235
|
+
beta,
|
|
236
|
+
average,
|
|
237
|
+
labels,
|
|
238
|
+
positiveLabel: null,
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* Formats a beta value for use in metric names, e.g. `1` -> `"f1"`,
|
|
243
|
+
* `0.5` -> `"f0_5"`.
|
|
244
|
+
*/
|
|
245
|
+
export function formatBetaForMetricName(beta) {
|
|
246
|
+
if (beta <= 0) {
|
|
247
|
+
return "f";
|
|
248
|
+
}
|
|
249
|
+
if (Number.isInteger(beta)) {
|
|
250
|
+
return `f${beta}`;
|
|
251
|
+
}
|
|
252
|
+
return `f${beta.toString().replace(".", "_")}`;
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* The suffix appended to a metric name to reflect the aggregation strategy,
|
|
256
|
+
* e.g. `"precision"` vs. `"precision_micro"`. No suffix is used when
|
|
257
|
+
* `positiveLabel` is explicitly configured, since `average` is not
|
|
258
|
+
* applicable in that one-vs-rest binary mode. This mirrors
|
|
259
|
+
* `resolvePositiveLabel`'s auto-detection rule (only under the default
|
|
260
|
+
* `"macro"` average) so a constructed evaluator's static name always
|
|
261
|
+
* matches what `computePrecisionRecallFScore` actually computes.
|
|
262
|
+
*/
|
|
263
|
+
export function getAverageMetricNameSuffix({ average = "macro", positiveLabel, }) {
|
|
264
|
+
if (positiveLabel !== undefined) {
|
|
265
|
+
return "";
|
|
266
|
+
}
|
|
267
|
+
return average === "macro" ? "" : `_${average}`;
|
|
268
|
+
}
|
|
269
|
+
//# sourceMappingURL=classificationMetrics.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"classificationMetrics.js","sourceRoot":"","sources":["../../../src/code/classificationMetrics.ts"],"names":[],"mappings":"AAmFA,SAAS,YAAY,CAAC,IAAY;IAChC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACtC,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,OAAoB;IAC3C,IAAI,OAAO,KAAK,OAAO,IAAI,OAAO,KAAK,OAAO,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC5E,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAC7B,QAA+B,EAC/B,MAA6B;IAE7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,sDAAsD,QAAQ,CAAC,MAAM,QAAQ,MAAM,CAAC,MAAM,EAAE,CAC7F,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CACjB,SAAiB,EACjB,WAAmB,EACnB,YAAoB;IAEpB,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,OAAO,SAAS,GAAG,WAAW,CAAC;AACjC,CAAC;AAED,SAAS,aAAa,CACpB,SAAiB,EACjB,MAAc,EACd,IAAY,EACZ,YAAoB;IAEpB,IAAI,SAAS,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,GAAG,IAAI,CAAC;IAChC,MAAM,SAAS,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC;IACzD,MAAM,WAAW,GAAG,WAAW,GAAG,SAAS,GAAG,MAAM,CAAC;IACrD,OAAO,UAAU,CAAC,SAAS,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;AAC1D,CAAC;AAED,2FAA2F;AAC3F,SAAS,aAAa,CACpB,QAA+B,EAC/B,MAA6B;IAE7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,MAAM,OAAO,GAA0B,EAAE,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CACrB,CAAsB,EACtB,CAAsB;IAEtB,OAAO,CACL,CAAC,KAAK,CAAC;QACP,CAAC,OAAO,CAAC,KAAK,QAAQ;YACpB,OAAO,CAAC,KAAK,QAAQ;YACrB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CACnB,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CACzB,QAA+B,EAC/B,MAA6B,EAC7B,MAA6B;IAE7B,MAAM,aAAa,GAAG,IAAI,GAAG,EAAoC,CAAC;IAClE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE;YACvB,YAAY,EAAE,CAAC;YACf,aAAa,EAAE,CAAC;YAChB,aAAa,EAAE,CAAC;SACjB,CAAC,CAAC;IACL,CAAC;IACD,KAAK,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACxD,+DAA+D;QAC/D,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAwB,CAAC;QACzD,0EAA0E;QAC1E,2DAA2D;QAC3D,IAAI,cAAc,CAAC,aAAa,EAAE,WAAW,CAAC,EAAE,CAAC;YAC/C,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,aAAa,CAAgB,CAAC;YAC/D,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,MAAM,eAAe,GAAG,aAAa,CAAC,GAAG,CAAC,WAAW,CAAgB,CAAC;YACtE,eAAe,CAAC,aAAa,IAAI,CAAC,CAAC;YACnC,MAAM,cAAc,GAAG,aAAa,CAAC,GAAG,CAAC,aAAa,CAAgB,CAAC;YACvE,cAAc,CAAC,aAAa,IAAI,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;;;GAKG;AACH,SAAS,oBAAoB,CAC3B,uBAAwD,EACxD,MAA6B,EAC7B,OAAoB;IAEpB,IAAI,uBAAuB,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,uBAAuB,CAAC;IACjC,CAAC;IACD,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACrC,IAAI,YAAY,CAAC,IAAI,KAAK,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,wBAAwB,CAC/B,aAAoD,EACpD,OAAoB,EACpB,IAAY,EACZ,YAAoB;IAEpB,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACxB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,KAAK,MAAM,MAAM,IAAI,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC;YACpC,aAAa,IAAI,MAAM,CAAC,aAAa,CAAC;YACtC,aAAa,IAAI,MAAM,CAAC,aAAa,CAAC;QACxC,CAAC;QACD,MAAM,SAAS,GAAG,UAAU,CAC1B,YAAY,EACZ,YAAY,GAAG,aAAa,EAC5B,YAAY,CACb,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,CACvB,YAAY,EACZ,YAAY,GAAG,aAAa,EAC5B,YAAY,CACb,CAAC;QACF,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC;SAC7D,CAAC;IACJ,CAAC;IAED,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACxE,MAAM,SAAS,GAAG,UAAU,CAC1B,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,aAAa,EAC1C,YAAY,CACb,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,CACvB,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,aAAa,EAC1C,YAAY,CACb,CAAC;QACF,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC;YAC5D,OAAO,EAAE,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,aAAa;SACpD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACxB,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACxD,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAClD,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;SACnD,CAAC;IACJ,CAAC;IAED,WAAW;IACX,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC5E,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO;YACL,SAAS,EAAE,YAAY;YACvB,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE,YAAY;SACrB,CAAC;IACJ,CAAC;IACD,OAAO;QACL,SAAS,EAAE,YAAY,CAAC,eAAe,EAAE,WAAW,EAAE,YAAY,CAAC;QACnE,MAAM,EAAE,YAAY,CAAC,eAAe,EAAE,QAAQ,EAAE,YAAY,CAAC;QAC7D,MAAM,EAAE,YAAY,CAAC,eAAe,EAAE,QAAQ,EAAE,YAAY,CAAC;KAC9D,CAAC;AACJ,CAAC;AAED,SAAS,IAAI,CAAC,MAAgB;IAC5B,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACvE,CAAC;AAED,SAAS,YAAY,CACnB,eAKE,EACF,KAAwC,EACxC,YAAoB;IAEpB,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CACxC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,EACxD,CAAC,CACF,CAAC;IACF,OAAO,WAAW,GAAG,YAAY,CAAC;AACpC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,UAAU,4BAA4B,CAC1C,EAAE,QAAQ,EAAE,MAAM,EAAsD,EACxE,UAAwC,EAAE;IAE1C,MAAM,EACJ,IAAI,GAAG,CAAC,EACR,OAAO,GAAG,OAAO,EACjB,YAAY,GAAG,CAAC,EAChB,aAAa,GACd,GAAG,OAAO,CAAC;IAEZ,YAAY,CAAC,IAAI,CAAC,CAAC;IACnB,eAAe,CAAC,OAAO,CAAC,CAAC;IACzB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEzC,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,aAAa,GAAG,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACnE,MAAM,qBAAqB,GAAG,oBAAoB,CAChD,aAAa,EACb,MAAM,EACN,OAAO,CACR,CAAC;IAEF,IAAI,qBAAqB,KAAK,IAAI,EAAE,CAAC;QACnC,MAAM,WAAW,GAAG,aAAa,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QAC7D,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACb,iBAAiB,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CACzG,CAAC;QACJ,CAAC;QACD,MAAM,SAAS,GAAG,UAAU,CAC1B,WAAW,CAAC,YAAY,EACxB,WAAW,CAAC,YAAY,GAAG,WAAW,CAAC,aAAa,EACpD,YAAY,CACb,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,CACvB,WAAW,CAAC,YAAY,EACxB,WAAW,CAAC,YAAY,GAAG,WAAW,CAAC,aAAa,EACpD,YAAY,CACb,CAAC;QACF,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC;YAC5D,IAAI;YACJ,OAAO;YACP,MAAM;YACN,aAAa,EAAE,qBAAqB;SACrC,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,wBAAwB,CAC5D,aAAa,EACb,OAAO,EACP,IAAI,EACJ,YAAY,CACb,CAAC;IACF,OAAO;QACL,SAAS;QACT,MAAM;QACN,MAAM;QACN,IAAI;QACJ,OAAO;QACP,MAAM;QACN,aAAa,EAAE,IAAI;KACpB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAY;IAClD,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACd,OAAO,GAAG,CAAC;IACb,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,OAAO,IAAI,IAAI,EAAE,CAAC;IACpB,CAAC;IACD,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;AACjD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,0BAA0B,CAAC,EACzC,OAAO,GAAG,OAAO,EACjB,aAAa,GACgB;IAC7B,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAChC,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,OAAO,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;AAClD,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { EvaluatorBase } from "../core/EvaluatorBase.js";
|
|
2
|
+
import { type ClassificationExample, type PrecisionRecallFScoreOptions, type PrecisionRecallFScoreResult } from "./classificationMetrics.js";
|
|
3
|
+
/**
|
|
4
|
+
* Computes a {@link PrecisionRecallFScoreResult} for a batch of labels.
|
|
5
|
+
* Injectable so callers (e.g. {@link createPrecisionRecallFScoreEvaluators})
|
|
6
|
+
* can share one computed result across multiple metric evaluators instead of
|
|
7
|
+
* recomputing it per evaluator.
|
|
8
|
+
*/
|
|
9
|
+
export type ClassificationMetricComputer = (example: Pick<ClassificationExample, "expected" | "output">, options: PrecisionRecallFScoreOptions) => PrecisionRecallFScoreResult;
|
|
10
|
+
/**
|
|
11
|
+
* Internal factory shared by `createPrecisionEvaluator`, `createRecallEvaluator`,
|
|
12
|
+
* and `createFBetaEvaluator` — each is a thin wrapper that only differs in
|
|
13
|
+
* which result field it reads and how its metric name is built.
|
|
14
|
+
*/
|
|
15
|
+
export declare function createClassificationMetricEvaluator<RecordType extends ClassificationExample = ClassificationExample>(name: string, field: "precision" | "recall" | "fScore", options: PrecisionRecallFScoreOptions, compute?: ClassificationMetricComputer): EvaluatorBase<RecordType>;
|
|
16
|
+
//# sourceMappingURL=createClassificationMetricEvaluator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createClassificationMetricEvaluator.d.ts","sourceRoot":"","sources":["../../../src/code/createClassificationMetricEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,EAEL,KAAK,qBAAqB,EAC1B,KAAK,4BAA4B,EACjC,KAAK,2BAA2B,EACjC,MAAM,yBAAyB,CAAC;AAEjC;;;;;GAKG;AACH,MAAM,MAAM,4BAA4B,GAAG,CACzC,OAAO,EAAE,IAAI,CAAC,qBAAqB,EAAE,UAAU,GAAG,QAAQ,CAAC,EAC3D,OAAO,EAAE,4BAA4B,KAClC,2BAA2B,CAAC;AAEjC;;;;GAIG;AACH,wBAAgB,mCAAmC,CACjD,UAAU,SAAS,qBAAqB,GAAG,qBAAqB,EAEhE,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,WAAW,GAAG,QAAQ,GAAG,QAAQ,EACxC,OAAO,EAAE,4BAA4B,EACrC,OAAO,GAAE,4BAA2D,GACnE,aAAa,CAAC,UAAU,CAAC,CAgB3B"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { createEvaluator } from "../helpers/createEvaluator.js";
|
|
2
|
+
import { computePrecisionRecallFScore, } from "./classificationMetrics.js";
|
|
3
|
+
/**
|
|
4
|
+
* Internal factory shared by `createPrecisionEvaluator`, `createRecallEvaluator`,
|
|
5
|
+
* and `createFBetaEvaluator` — each is a thin wrapper that only differs in
|
|
6
|
+
* which result field it reads and how its metric name is built.
|
|
7
|
+
*/
|
|
8
|
+
export function createClassificationMetricEvaluator(name, field, options, compute = computePrecisionRecallFScore) {
|
|
9
|
+
return createEvaluator(
|
|
10
|
+
// Pass `example` through by reference (rather than destructuring and
|
|
11
|
+
// rebuilding a new object) so callers that share one computed result by
|
|
12
|
+
// caching on object identity (e.g. `createPrecisionRecallFScoreEvaluators`)
|
|
13
|
+
// actually get a cache hit.
|
|
14
|
+
(example) => {
|
|
15
|
+
const result = compute(example, options);
|
|
16
|
+
return { score: result[field] };
|
|
17
|
+
}, {
|
|
18
|
+
name,
|
|
19
|
+
kind: "CODE",
|
|
20
|
+
optimizationDirection: "MAXIMIZE",
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=createClassificationMetricEvaluator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createClassificationMetricEvaluator.js","sourceRoot":"","sources":["../../../src/code/createClassificationMetricEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EACL,4BAA4B,GAI7B,MAAM,yBAAyB,CAAC;AAajC;;;;GAIG;AACH,MAAM,UAAU,mCAAmC,CAGjD,IAAY,EACZ,KAAwC,EACxC,OAAqC,EACrC,UAAwC,4BAA4B;IAEpE,OAAO,eAAe;IACpB,qEAAqE;IACrE,wEAAwE;IACxE,4EAA4E;IAC5E,4BAA4B;IAC5B,CAAC,OAAO,EAAE,EAAE;QACV,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IAClC,CAAC,EACD;QACE,IAAI;QACJ,IAAI,EAAE,MAAM;QACZ,qBAAqB,EAAE,UAAU;KAClC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { EvaluatorBase } from "../core/EvaluatorBase.js";
|
|
2
|
+
import type { ClassificationExample, PrecisionRecallFScoreOptions } from "./classificationMetrics.js";
|
|
3
|
+
export type F1EvaluatorOptions = Omit<PrecisionRecallFScoreOptions, "beta">;
|
|
4
|
+
/**
|
|
5
|
+
* Creates a code evaluator that computes the F1 score: the harmonic mean of
|
|
6
|
+
* precision and recall.
|
|
7
|
+
*
|
|
8
|
+
* Supports binary classification (via `positiveLabel`, or auto-detected when
|
|
9
|
+
* labels are the numeric set `{0, 1}`) and multi-class classification (via
|
|
10
|
+
* the `average` strategy).
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```typescript
|
|
14
|
+
* const f1 = createF1Evaluator();
|
|
15
|
+
* const result = await f1.evaluate({
|
|
16
|
+
* expected: ["cat", "dog", "cat", "bird"],
|
|
17
|
+
* output: ["cat", "cat", "cat", "bird"],
|
|
18
|
+
* });
|
|
19
|
+
* // { score: 0.6 }
|
|
20
|
+
* ```
|
|
21
|
+
*/
|
|
22
|
+
export declare function createF1Evaluator<RecordType extends ClassificationExample = ClassificationExample>(options?: F1EvaluatorOptions): EvaluatorBase<RecordType>;
|
|
23
|
+
//# sourceMappingURL=createF1Evaluator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createF1Evaluator.d.ts","sourceRoot":"","sources":["../../../src/code/createF1Evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EACV,qBAAqB,EACrB,4BAA4B,EAC7B,MAAM,yBAAyB,CAAC;AAGjC,MAAM,MAAM,kBAAkB,GAAG,IAAI,CAAC,4BAA4B,EAAE,MAAM,CAAC,CAAC;AAE5E;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,iBAAiB,CAC/B,UAAU,SAAS,qBAAqB,GAAG,qBAAqB,EAChE,OAAO,GAAE,kBAAuB,GAAG,aAAa,CAAC,UAAU,CAAC,CAE7D"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { createFBetaEvaluator } from "./createFBetaEvaluator.js";
|
|
2
|
+
/**
|
|
3
|
+
* Creates a code evaluator that computes the F1 score: the harmonic mean of
|
|
4
|
+
* precision and recall.
|
|
5
|
+
*
|
|
6
|
+
* Supports binary classification (via `positiveLabel`, or auto-detected when
|
|
7
|
+
* labels are the numeric set `{0, 1}`) and multi-class classification (via
|
|
8
|
+
* the `average` strategy).
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```typescript
|
|
12
|
+
* const f1 = createF1Evaluator();
|
|
13
|
+
* const result = await f1.evaluate({
|
|
14
|
+
* expected: ["cat", "dog", "cat", "bird"],
|
|
15
|
+
* output: ["cat", "cat", "cat", "bird"],
|
|
16
|
+
* });
|
|
17
|
+
* // { score: 0.6 }
|
|
18
|
+
* ```
|
|
19
|
+
*/
|
|
20
|
+
export function createF1Evaluator(options = {}) {
|
|
21
|
+
return createFBetaEvaluator({ ...options, beta: 1 });
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=createF1Evaluator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createF1Evaluator.js","sourceRoot":"","sources":["../../../src/code/createF1Evaluator.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAI9D;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,iBAAiB,CAE/B,UAA8B,EAAE;IAChC,OAAO,oBAAoB,CAAa,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;AACnE,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { EvaluatorBase } from "../core/EvaluatorBase.js";
|
|
2
|
+
import type { ClassificationExample, PrecisionRecallFScoreOptions } from "./classificationMetrics.js";
|
|
3
|
+
/**
|
|
4
|
+
* Creates a code evaluator that computes the F-beta score: the weighted
|
|
5
|
+
* harmonic mean of precision and recall, where `beta` controls how much more
|
|
6
|
+
* weight recall gets relative to precision (`beta = 1` is the standard F1
|
|
7
|
+
* score).
|
|
8
|
+
*
|
|
9
|
+
* Supports binary classification (via `positiveLabel`, or auto-detected when
|
|
10
|
+
* `average` is at its default `"macro"` and labels are the numeric set
|
|
11
|
+
* `{0, 1}`) and multi-class classification (via the `average` strategy).
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```typescript
|
|
15
|
+
* const f2 = createFBetaEvaluator({ beta: 2 });
|
|
16
|
+
* const result = await f2.evaluate({
|
|
17
|
+
* expected: ["cat", "dog", "cat", "bird"],
|
|
18
|
+
* output: ["cat", "cat", "cat", "bird"],
|
|
19
|
+
* });
|
|
20
|
+
* ```
|
|
21
|
+
*/
|
|
22
|
+
export declare function createFBetaEvaluator<RecordType extends ClassificationExample = ClassificationExample>(options?: PrecisionRecallFScoreOptions): EvaluatorBase<RecordType>;
|
|
23
|
+
//# sourceMappingURL=createFBetaEvaluator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createFBetaEvaluator.d.ts","sourceRoot":"","sources":["../../../src/code/createFBetaEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAK3D,OAAO,KAAK,EACV,qBAAqB,EACrB,4BAA4B,EAC7B,MAAM,yBAAyB,CAAC;AAGjC;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,oBAAoB,CAClC,UAAU,SAAS,qBAAqB,GAAG,qBAAqB,EAChE,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,UAAU,CAAC,CAQvE"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { formatBetaForMetricName, getAverageMetricNameSuffix, } from "./classificationMetrics.js";
|
|
2
|
+
import { createClassificationMetricEvaluator } from "./createClassificationMetricEvaluator.js";
|
|
3
|
+
/**
|
|
4
|
+
* Creates a code evaluator that computes the F-beta score: the weighted
|
|
5
|
+
* harmonic mean of precision and recall, where `beta` controls how much more
|
|
6
|
+
* weight recall gets relative to precision (`beta = 1` is the standard F1
|
|
7
|
+
* score).
|
|
8
|
+
*
|
|
9
|
+
* Supports binary classification (via `positiveLabel`, or auto-detected when
|
|
10
|
+
* `average` is at its default `"macro"` and labels are the numeric set
|
|
11
|
+
* `{0, 1}`) and multi-class classification (via the `average` strategy).
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```typescript
|
|
15
|
+
* const f2 = createFBetaEvaluator({ beta: 2 });
|
|
16
|
+
* const result = await f2.evaluate({
|
|
17
|
+
* expected: ["cat", "dog", "cat", "bird"],
|
|
18
|
+
* output: ["cat", "cat", "cat", "bird"],
|
|
19
|
+
* });
|
|
20
|
+
* ```
|
|
21
|
+
*/
|
|
22
|
+
export function createFBetaEvaluator(options = {}) {
|
|
23
|
+
const { beta = 1 } = options;
|
|
24
|
+
const suffix = getAverageMetricNameSuffix(options);
|
|
25
|
+
return createClassificationMetricEvaluator(`${formatBetaForMetricName(beta)}${suffix}`, "fScore", options);
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=createFBetaEvaluator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createFBetaEvaluator.js","sourceRoot":"","sources":["../../../src/code/createFBetaEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EACL,uBAAuB,EACvB,0BAA0B,GAC3B,MAAM,yBAAyB,CAAC;AAKjC,OAAO,EAAE,mCAAmC,EAAE,MAAM,uCAAuC,CAAC;AAE5F;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,oBAAoB,CAElC,UAAwC,EAAE;IAC1C,MAAM,EAAE,IAAI,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAC7B,MAAM,MAAM,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACnD,OAAO,mCAAmC,CACxC,GAAG,uBAAuB,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,EAC3C,QAAQ,EACR,OAAO,CACR,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { EvaluatorBase } from "../core/EvaluatorBase.js";
|
|
2
|
+
import type { ClassificationExample, PrecisionRecallFScoreOptions } from "./classificationMetrics.js";
|
|
3
|
+
/**
|
|
4
|
+
* Creates a code evaluator that computes precision: of the labels the model
|
|
5
|
+
* predicted as a given class, the fraction that were actually that class.
|
|
6
|
+
*
|
|
7
|
+
* Supports binary classification (via `positiveLabel`, or auto-detected when
|
|
8
|
+
* `average` is at its default `"macro"` and labels are the numeric set
|
|
9
|
+
* `{0, 1}`) and multi-class classification (via the `average` strategy).
|
|
10
|
+
*
|
|
11
|
+
* @example
|
|
12
|
+
* ```typescript
|
|
13
|
+
* const precision = createPrecisionEvaluator();
|
|
14
|
+
* const result = await precision.evaluate({
|
|
15
|
+
* expected: ["cat", "dog", "cat", "bird"],
|
|
16
|
+
* output: ["cat", "cat", "cat", "bird"],
|
|
17
|
+
* });
|
|
18
|
+
* // { score: 5/9 }
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
export declare function createPrecisionEvaluator<RecordType extends ClassificationExample = ClassificationExample>(options?: PrecisionRecallFScoreOptions): EvaluatorBase<RecordType>;
|
|
22
|
+
//# sourceMappingURL=createPrecisionEvaluator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createPrecisionEvaluator.d.ts","sourceRoot":"","sources":["../../../src/code/createPrecisionEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,KAAK,EACV,qBAAqB,EACrB,4BAA4B,EAC7B,MAAM,yBAAyB,CAAC;AAGjC;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,wBAAwB,CACtC,UAAU,SAAS,qBAAqB,GAAG,qBAAqB,EAChE,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,UAAU,CAAC,CAOvE"}
|