@arizeai/phoenix-evals 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -0
- package/dist/esm/code/classificationMetrics.d.ts +121 -0
- package/dist/esm/code/classificationMetrics.d.ts.map +1 -0
- package/dist/esm/code/classificationMetrics.js +269 -0
- package/dist/esm/code/classificationMetrics.js.map +1 -0
- package/dist/esm/code/createClassificationMetricEvaluator.d.ts +16 -0
- package/dist/esm/code/createClassificationMetricEvaluator.d.ts.map +1 -0
- package/dist/esm/code/createClassificationMetricEvaluator.js +23 -0
- package/dist/esm/code/createClassificationMetricEvaluator.js.map +1 -0
- package/dist/esm/code/createF1Evaluator.d.ts +23 -0
- package/dist/esm/code/createF1Evaluator.d.ts.map +1 -0
- package/dist/esm/code/createF1Evaluator.js +23 -0
- package/dist/esm/code/createF1Evaluator.js.map +1 -0
- package/dist/esm/code/createFBetaEvaluator.d.ts +23 -0
- package/dist/esm/code/createFBetaEvaluator.d.ts.map +1 -0
- package/dist/esm/code/createFBetaEvaluator.js +27 -0
- package/dist/esm/code/createFBetaEvaluator.js.map +1 -0
- package/dist/esm/code/createPrecisionEvaluator.d.ts +22 -0
- package/dist/esm/code/createPrecisionEvaluator.d.ts.map +1 -0
- package/dist/esm/code/createPrecisionEvaluator.js +25 -0
- package/dist/esm/code/createPrecisionEvaluator.js.map +1 -0
- package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
- package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
- package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js +44 -0
- package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
- package/dist/esm/code/createRecallEvaluator.d.ts +22 -0
- package/dist/esm/code/createRecallEvaluator.d.ts.map +1 -0
- package/dist/esm/code/createRecallEvaluator.js +25 -0
- package/dist/esm/code/createRecallEvaluator.js.map +1 -0
- package/dist/esm/code/index.d.ts +7 -0
- package/dist/esm/code/index.d.ts.map +1 -0
- package/dist/esm/code/index.js +7 -0
- package/dist/esm/code/index.js.map +1 -0
- package/dist/esm/index.d.ts +1 -0
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +1 -0
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/src/code/classificationMetrics.d.ts +121 -0
- package/dist/src/code/classificationMetrics.d.ts.map +1 -0
- package/dist/src/code/classificationMetrics.js +274 -0
- package/dist/src/code/classificationMetrics.js.map +1 -0
- package/dist/src/code/createClassificationMetricEvaluator.d.ts +16 -0
- package/dist/src/code/createClassificationMetricEvaluator.d.ts.map +1 -0
- package/dist/src/code/createClassificationMetricEvaluator.js +26 -0
- package/dist/src/code/createClassificationMetricEvaluator.js.map +1 -0
- package/dist/src/code/createF1Evaluator.d.ts +23 -0
- package/dist/src/code/createF1Evaluator.d.ts.map +1 -0
- package/dist/src/code/createF1Evaluator.js +26 -0
- package/dist/src/code/createF1Evaluator.js.map +1 -0
- package/dist/src/code/createFBetaEvaluator.d.ts +23 -0
- package/dist/src/code/createFBetaEvaluator.d.ts.map +1 -0
- package/dist/src/code/createFBetaEvaluator.js +30 -0
- package/dist/src/code/createFBetaEvaluator.js.map +1 -0
- package/dist/src/code/createPrecisionEvaluator.d.ts +22 -0
- package/dist/src/code/createPrecisionEvaluator.d.ts.map +1 -0
- package/dist/src/code/createPrecisionEvaluator.js +28 -0
- package/dist/src/code/createPrecisionEvaluator.js.map +1 -0
- package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
- package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
- package/dist/src/code/createPrecisionRecallFScoreEvaluators.js +47 -0
- package/dist/src/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
- package/dist/src/code/createRecallEvaluator.d.ts +22 -0
- package/dist/src/code/createRecallEvaluator.d.ts.map +1 -0
- package/dist/src/code/createRecallEvaluator.js +28 -0
- package/dist/src/code/createRecallEvaluator.js.map +1 -0
- package/dist/src/code/index.d.ts +7 -0
- package/dist/src/code/index.d.ts.map +1 -0
- package/dist/src/code/index.js +23 -0
- package/dist/src/code/index.js.map +1 -0
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -0
- package/dist/src/index.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/docs/classification-metrics.mdx +103 -0
- package/docs/overview.mdx +4 -1
- package/package.json +24 -16
- package/src/code/classificationMetrics.ts +464 -0
- package/src/code/createClassificationMetricEvaluator.ts +49 -0
- package/src/code/createF1Evaluator.ts +32 -0
- package/src/code/createFBetaEvaluator.ts +41 -0
- package/src/code/createPrecisionEvaluator.ts +36 -0
- package/src/code/createPrecisionRecallFScoreEvaluators.ts +83 -0
- package/src/code/createRecallEvaluator.ts +36 -0
- package/src/code/index.ts +6 -0
- package/src/index.ts +1 -0
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Classification Metrics"
|
|
3
|
+
description: "Precision, recall, and F-beta code evaluators in @arizeai/phoenix-evals"
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
The `code` entrypoint provides deterministic, non-LLM evaluators for classification metrics: precision, recall, and F-beta (including F1). Unlike the per-row LLM evaluators elsewhere in this package, these are dataset-level — `expected` and `output` are the full sequence of labels across every example you want to score together, not a single row's label.
|
|
7
|
+
|
|
8
|
+
<section className="hidden" data-agent-context="relevant-source-files" aria-label="Relevant source files">
|
|
9
|
+
<h2>Relevant Source Files</h2>
|
|
10
|
+
<ul>
|
|
11
|
+
<li><code>src/code/index.ts</code></li>
|
|
12
|
+
<li><code>src/code/classificationMetrics.ts</code></li>
|
|
13
|
+
</ul>
|
|
14
|
+
</section>
|
|
15
|
+
|
|
16
|
+
## Example
|
|
17
|
+
|
|
18
|
+
```ts
|
|
19
|
+
import { createF1Evaluator } from "@arizeai/phoenix-evals/code";
|
|
20
|
+
|
|
21
|
+
const f1 = createF1Evaluator();
|
|
22
|
+
|
|
23
|
+
const result = await f1.evaluate({
|
|
24
|
+
expected: ["cat", "dog", "cat", "bird"],
|
|
25
|
+
output: ["cat", "cat", "cat", "bird"],
|
|
26
|
+
});
|
|
27
|
+
// { score: 0.6 }
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Built-In Evaluator Factories
|
|
31
|
+
|
|
32
|
+
- `createPrecisionEvaluator(options?)` — `score` is precision
|
|
33
|
+
- `createRecallEvaluator(options?)` — `score` is recall
|
|
34
|
+
- `createF1Evaluator(options?)` — `score` is F1 (F-beta with `beta: 1`)
|
|
35
|
+
- `createFBetaEvaluator(options?)` — `score` is F-beta; set `beta` to weight recall (`beta > 1`) or precision (`beta < 1`) more heavily
|
|
36
|
+
- `createPrecisionRecallFScoreEvaluators(options?)` — returns `{ precision, recall, fScore }`: three separate evaluators built from one shared options object
|
|
37
|
+
|
|
38
|
+
Each evaluator resolves to a single `{ score }`. (Python's `PrecisionRecallFScore` returns all three scores from one call; in TypeScript each evaluator yields one score, so use the bundle above to get all three.) Each factory accepts the same options:
|
|
39
|
+
|
|
40
|
+
```ts
|
|
41
|
+
interface PrecisionRecallFScoreOptions {
|
|
42
|
+
beta?: number; // default 1
|
|
43
|
+
average?: "macro" | "micro" | "weighted"; // default "macro"
|
|
44
|
+
positiveLabel?: string | number; // for binary one-vs-rest scoring
|
|
45
|
+
zeroDivision?: number; // default 0, used when a metric is undefined (0/0)
|
|
46
|
+
}
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Binary Classification
|
|
50
|
+
|
|
51
|
+
```ts
|
|
52
|
+
import {
|
|
53
|
+
createPrecisionEvaluator,
|
|
54
|
+
createRecallEvaluator,
|
|
55
|
+
} from "@arizeai/phoenix-evals/code";
|
|
56
|
+
|
|
57
|
+
const precision = createPrecisionEvaluator({ positiveLabel: "spam" });
|
|
58
|
+
const recall = createRecallEvaluator({ positiveLabel: "spam" });
|
|
59
|
+
|
|
60
|
+
const example = {
|
|
61
|
+
expected: ["spam", "ham", "spam", "ham", "spam"],
|
|
62
|
+
output: ["spam", "spam", "ham", "ham", "spam"],
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
await precision.evaluate(example); // { score: 0.667 }
|
|
66
|
+
await recall.evaluate(example); // { score: 0.667 }
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Multi-Class Averaging
|
|
70
|
+
|
|
71
|
+
```ts
|
|
72
|
+
import { createPrecisionRecallFScoreEvaluators } from "@arizeai/phoenix-evals/code";
|
|
73
|
+
|
|
74
|
+
// "macro" (default): unweighted mean across classes
|
|
75
|
+
// "weighted": mean weighted by each class's support
|
|
76
|
+
// "micro": pool TP/FP/FN across classes first (equals accuracy for
|
|
77
|
+
// single-label multi-class problems)
|
|
78
|
+
const { precision, recall, fScore } = createPrecisionRecallFScoreEvaluators({
|
|
79
|
+
average: "weighted",
|
|
80
|
+
});
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## When To Use
|
|
84
|
+
|
|
85
|
+
- Scoring classification or labeling tasks (routing, tagging, moderation categories) against ground-truth labels
|
|
86
|
+
- Comparing predicted vs. expected label sequences collected from an experiment or dataset
|
|
87
|
+
- Choosing an F-beta weighting that matches the cost of false positives vs. false negatives for your task (e.g. `beta: 2` when missing a true positive is costlier than a false alarm)
|
|
88
|
+
|
|
89
|
+
For the underlying formulas, the precision/recall tradeoff, averaging-strategy semantics, and citations, see the [Precision / Recall / F-Score](/docs/phoenix/evaluation/pre-built-metrics/precision-recall-fscore) guide. A runnable walkthrough covering binary classification, F-beta tradeoffs, and all three averaging strategies lives in the package's [`examples/classification_metrics_example.ts`](https://github.com/Arize-ai/phoenix/blob/main/js/packages/phoenix-evals/examples/classification_metrics_example.ts).
|
|
90
|
+
|
|
91
|
+
<section className="hidden" data-agent-context="source-map" aria-label="Source map">
|
|
92
|
+
<h2>Source Map</h2>
|
|
93
|
+
<ul>
|
|
94
|
+
<li><code>src/code/index.ts</code></li>
|
|
95
|
+
<li><code>src/code/classificationMetrics.ts</code></li>
|
|
96
|
+
<li><code>src/code/createClassificationMetricEvaluator.ts</code></li>
|
|
97
|
+
<li><code>src/code/createPrecisionEvaluator.ts</code></li>
|
|
98
|
+
<li><code>src/code/createRecallEvaluator.ts</code></li>
|
|
99
|
+
<li><code>src/code/createFBetaEvaluator.ts</code></li>
|
|
100
|
+
<li><code>src/code/createF1Evaluator.ts</code></li>
|
|
101
|
+
<li><code>src/code/createPrecisionRecallFScoreEvaluators.ts</code></li>
|
|
102
|
+
</ul>
|
|
103
|
+
</section>
|
package/docs/overview.mdx
CHANGED
|
@@ -61,18 +61,20 @@ node_modules/@arizeai/phoenix-evals/docs/
|
|
|
61
61
|
node_modules/@arizeai/phoenix-evals/src/
|
|
62
62
|
```
|
|
63
63
|
|
|
64
|
-
The bundled docs cover evaluator creation, LLM evaluators, templates, classification, and Phoenix integration.
|
|
64
|
+
The bundled docs cover evaluator creation, LLM evaluators, classification metrics, templates, classification, and Phoenix integration.
|
|
65
65
|
|
|
66
66
|
## Where To Start
|
|
67
67
|
|
|
68
68
|
- [Create evaluator](./create-evaluator) for custom and code-based evaluator flows
|
|
69
69
|
- [LLM evaluators](./llm-evaluators) and [Classification](./classification) for model-backed evaluation
|
|
70
|
+
- [Classification metrics](./classification-metrics) for precision/recall/F-beta code evaluators
|
|
70
71
|
- [Templates](./templates) and [Phoenix integration](./phoenix-integration) for prompt helpers and experiment wiring
|
|
71
72
|
|
|
72
73
|
## Source Layout
|
|
73
74
|
|
|
74
75
|
- `src/index.ts` re-exports the package surface you usually import from `@arizeai/phoenix-evals`
|
|
75
76
|
- `src/llm/` contains classification helpers and built-in LLM evaluator factories
|
|
77
|
+
- `src/code/` contains deterministic classification-metric evaluators (precision, recall, F-beta)
|
|
76
78
|
- `src/helpers/` contains `createEvaluator` and evaluation-result helpers
|
|
77
79
|
- `src/template/` contains `formatTemplate` and `getTemplateVariables`
|
|
78
80
|
- `src/types/` contains shared evaluator and prompt types
|
|
@@ -82,6 +84,7 @@ The bundled docs cover evaluator creation, LLM evaluators, templates, classifica
|
|
|
82
84
|
<ul>
|
|
83
85
|
<li><code>src/index.ts</code></li>
|
|
84
86
|
<li><code>src/llm/</code></li>
|
|
87
|
+
<li><code>src/code/</code></li>
|
|
85
88
|
<li><code>src/helpers/</code></li>
|
|
86
89
|
<li><code>src/template/</code></li>
|
|
87
90
|
<li><code>src/core/</code></li>
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@arizeai/phoenix-evals",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "A library for running evaluations for AI use cases",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"arize",
|
|
@@ -40,6 +40,14 @@
|
|
|
40
40
|
"import": "./dist/esm/llm/*.js",
|
|
41
41
|
"require": "./dist/src/llm/*.js"
|
|
42
42
|
},
|
|
43
|
+
"./code": {
|
|
44
|
+
"import": "./dist/esm/code/index.js",
|
|
45
|
+
"require": "./dist/src/code/index.js"
|
|
46
|
+
},
|
|
47
|
+
"./code/*": {
|
|
48
|
+
"import": "./dist/esm/code/*.js",
|
|
49
|
+
"require": "./dist/src/code/*.js"
|
|
50
|
+
},
|
|
43
51
|
"./template/*": {
|
|
44
52
|
"import": "./dist/esm/template/*.js",
|
|
45
53
|
"require": "./dist/src/template/*.js"
|
|
@@ -58,25 +66,25 @@
|
|
|
58
66
|
}
|
|
59
67
|
},
|
|
60
68
|
"dependencies": {
|
|
61
|
-
"@arizeai/openinference-core": "^2.
|
|
62
|
-
"@opentelemetry/api": "^1.9.
|
|
63
|
-
"ai": "^6.0.
|
|
64
|
-
"jsonpath-plus": "^10.
|
|
69
|
+
"@arizeai/openinference-core": "^2.4.0",
|
|
70
|
+
"@opentelemetry/api": "^1.9.1",
|
|
71
|
+
"ai": "^6.0.211",
|
|
72
|
+
"jsonpath-plus": "^10.4.0",
|
|
65
73
|
"mustache": "^4.2.0",
|
|
66
|
-
"zod": "^4.
|
|
74
|
+
"zod": "^4.4.3"
|
|
67
75
|
},
|
|
68
76
|
"devDependencies": {
|
|
69
|
-
"@ai-sdk/google": "^3.0.
|
|
70
|
-
"@ai-sdk/openai": "^3.0.
|
|
71
|
-
"@arizeai/openinference-instrumentation-openai": "^
|
|
72
|
-
"@opentelemetry/sdk-trace-node": "^
|
|
77
|
+
"@ai-sdk/google": "^3.0.84",
|
|
78
|
+
"@ai-sdk/openai": "^3.0.75",
|
|
79
|
+
"@arizeai/openinference-instrumentation-openai": "^4.1.4",
|
|
80
|
+
"@opentelemetry/sdk-trace-node": "^2.8.0",
|
|
73
81
|
"@types/mustache": "^4.2.6",
|
|
74
|
-
"@types/node": "^
|
|
75
|
-
"msw": "^2.
|
|
76
|
-
"nock": "^14.0.
|
|
77
|
-
"tsx": "^4.
|
|
78
|
-
"typedoc": "^0.28.
|
|
79
|
-
"vitest": "^4.1.
|
|
82
|
+
"@types/node": "^25.9.4",
|
|
83
|
+
"msw": "^2.14.6",
|
|
84
|
+
"nock": "^14.0.16",
|
|
85
|
+
"tsx": "^4.22.4",
|
|
86
|
+
"typedoc": "^0.28.19",
|
|
87
|
+
"vitest": "^4.1.9"
|
|
80
88
|
},
|
|
81
89
|
"engines": {
|
|
82
90
|
"node": ">=18"
|
|
@@ -0,0 +1,464 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A label produced by a classifier. Must be a primitive so it can be compared
|
|
3
|
+
* for equality and used as a lookup key.
|
|
4
|
+
*/
|
|
5
|
+
export type ClassificationLabel = string | number;
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The strategy used to aggregate per-class precision/recall/F-score into a
|
|
9
|
+
* single number when there are more than two classes (or no `positiveLabel`
|
|
10
|
+
* is configured).
|
|
11
|
+
*
|
|
12
|
+
* - `"macro"`: unweighted mean across classes.
|
|
13
|
+
* - `"micro"`: pool true/false positives and false negatives across classes
|
|
14
|
+
* before computing the metric.
|
|
15
|
+
* - `"weighted"`: mean across classes, weighted by each class's support
|
|
16
|
+
* (number of true instances).
|
|
17
|
+
*/
|
|
18
|
+
export type AverageType = "macro" | "micro" | "weighted";
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* The example shape expected by the classification-metric evaluators.
|
|
22
|
+
*/
|
|
23
|
+
export interface ClassificationExample {
|
|
24
|
+
/** The ground-truth sequence of labels. */
|
|
25
|
+
expected: ClassificationLabel[];
|
|
26
|
+
/** The predicted sequence of labels, aligned by index with `expected`. */
|
|
27
|
+
output: ClassificationLabel[];
|
|
28
|
+
[key: string]: unknown;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Options shared by precision, recall, and F-score computations.
|
|
33
|
+
*/
|
|
34
|
+
export interface PrecisionRecallFScoreOptions {
|
|
35
|
+
/**
|
|
36
|
+
* Weight of recall relative to precision in the F-score. Must be > 0.
|
|
37
|
+
* @defaultValue 1
|
|
38
|
+
*/
|
|
39
|
+
beta?: number;
|
|
40
|
+
/**
|
|
41
|
+
* Aggregation strategy across classes. Ignored when `positiveLabel` is set
|
|
42
|
+
* (or auto-detected).
|
|
43
|
+
* @defaultValue "macro"
|
|
44
|
+
*/
|
|
45
|
+
average?: AverageType;
|
|
46
|
+
/**
|
|
47
|
+
* Value substituted for a metric when it is undefined (e.g. 0/0).
|
|
48
|
+
* @defaultValue 0
|
|
49
|
+
*/
|
|
50
|
+
zeroDivision?: number;
|
|
51
|
+
/**
|
|
52
|
+
* When set, compute binary precision/recall/F exclusively for this label
|
|
53
|
+
* (one-vs-rest). If not set, `average` is at its default `"macro"`, and
|
|
54
|
+
* the labels are the numeric set `{0, 1}`, the positive label defaults to
|
|
55
|
+
* `1`. Otherwise, multi-class averaging is used. The auto-detection is
|
|
56
|
+
* skipped whenever a non-default `average` is configured, so an explicit
|
|
57
|
+
* `average` is never silently overridden by the shape of the data.
|
|
58
|
+
*/
|
|
59
|
+
positiveLabel?: ClassificationLabel;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* The result of computing precision, recall, and F-score for a batch of
|
|
64
|
+
* predictions.
|
|
65
|
+
*/
|
|
66
|
+
export interface PrecisionRecallFScoreResult {
|
|
67
|
+
precision: number;
|
|
68
|
+
recall: number;
|
|
69
|
+
fScore: number;
|
|
70
|
+
beta: number;
|
|
71
|
+
average: AverageType;
|
|
72
|
+
/** All labels observed in `expected` and `output`, in first-seen order. */
|
|
73
|
+
labels: ClassificationLabel[];
|
|
74
|
+
/** The label treated as positive in one-vs-rest mode, or `null` if multi-class averaging was used. */
|
|
75
|
+
positiveLabel: ClassificationLabel | null;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
interface ClassCounts {
|
|
79
|
+
truePositive: number;
|
|
80
|
+
falsePositive: number;
|
|
81
|
+
falseNegative: number;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function validateBeta(beta: number): void {
|
|
85
|
+
if (!(beta > 0)) {
|
|
86
|
+
throw new Error("beta must be > 0");
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function validateAverage(average: AverageType): void {
|
|
91
|
+
if (average !== "macro" && average !== "micro" && average !== "weighted") {
|
|
92
|
+
throw new Error('average must be one of "macro", "micro", or "weighted"');
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function validateLabelSequences(
|
|
97
|
+
expected: ClassificationLabel[],
|
|
98
|
+
output: ClassificationLabel[]
|
|
99
|
+
): void {
|
|
100
|
+
if (!Array.isArray(expected) || !Array.isArray(output)) {
|
|
101
|
+
throw new Error("expected and output must be arrays of labels");
|
|
102
|
+
}
|
|
103
|
+
if (expected.length !== output.length) {
|
|
104
|
+
throw new Error(
|
|
105
|
+
`expected and output must have the same length. Got ${expected.length} and ${output.length}`
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
if (expected.length === 0) {
|
|
109
|
+
throw new Error("expected and output must be non-empty");
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function safeDivide(
|
|
114
|
+
numerator: number,
|
|
115
|
+
denominator: number,
|
|
116
|
+
zeroDivision: number
|
|
117
|
+
): number {
|
|
118
|
+
if (denominator === 0) {
|
|
119
|
+
return zeroDivision;
|
|
120
|
+
}
|
|
121
|
+
return numerator / denominator;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function computeFScore(
|
|
125
|
+
precision: number,
|
|
126
|
+
recall: number,
|
|
127
|
+
beta: number,
|
|
128
|
+
zeroDivision: number
|
|
129
|
+
): number {
|
|
130
|
+
if (precision === 0 && recall === 0) {
|
|
131
|
+
return 0;
|
|
132
|
+
}
|
|
133
|
+
const betaSquared = beta * beta;
|
|
134
|
+
const numerator = (1 + betaSquared) * precision * recall;
|
|
135
|
+
const denominator = betaSquared * precision + recall;
|
|
136
|
+
return safeDivide(numerator, denominator, zeroDivision);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Collects all unique labels, ordered by first appearance in `expected` then `output`. */
|
|
140
|
+
function collectLabels(
|
|
141
|
+
expected: ClassificationLabel[],
|
|
142
|
+
output: ClassificationLabel[]
|
|
143
|
+
): ClassificationLabel[] {
|
|
144
|
+
const seen = new Set<ClassificationLabel>();
|
|
145
|
+
const ordered: ClassificationLabel[] = [];
|
|
146
|
+
for (const label of [...expected, ...output]) {
|
|
147
|
+
if (!seen.has(label)) {
|
|
148
|
+
seen.add(label);
|
|
149
|
+
ordered.push(label);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return ordered;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Compares two labels the same way `Map`/`Set` key lookups do (SameValueZero),
|
|
157
|
+
* so a `NaN` label matches another `NaN` label instead of always mismatching
|
|
158
|
+
* under strict `===`.
|
|
159
|
+
*/
|
|
160
|
+
function labelsAreEqual(
|
|
161
|
+
a: ClassificationLabel,
|
|
162
|
+
b: ClassificationLabel
|
|
163
|
+
): boolean {
|
|
164
|
+
return (
|
|
165
|
+
a === b ||
|
|
166
|
+
(typeof a === "number" &&
|
|
167
|
+
typeof b === "number" &&
|
|
168
|
+
Number.isNaN(a) &&
|
|
169
|
+
Number.isNaN(b))
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function computeClassCounts(
|
|
174
|
+
expected: ClassificationLabel[],
|
|
175
|
+
output: ClassificationLabel[],
|
|
176
|
+
labels: ClassificationLabel[]
|
|
177
|
+
): Map<ClassificationLabel, ClassCounts> {
|
|
178
|
+
const countsByLabel = new Map<ClassificationLabel, ClassCounts>();
|
|
179
|
+
for (const label of labels) {
|
|
180
|
+
countsByLabel.set(label, {
|
|
181
|
+
truePositive: 0,
|
|
182
|
+
falsePositive: 0,
|
|
183
|
+
falseNegative: 0,
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
for (const [index, expectedLabel] of expected.entries()) {
|
|
187
|
+
// `output` is validated to have the same length as `expected`.
|
|
188
|
+
const outputLabel = output[index] as ClassificationLabel;
|
|
189
|
+
// `labels` is derived from these same `expected`/`output` arrays, so both
|
|
190
|
+
// labels are always pre-populated keys in `countsByLabel`.
|
|
191
|
+
if (labelsAreEqual(expectedLabel, outputLabel)) {
|
|
192
|
+
const counts = countsByLabel.get(expectedLabel) as ClassCounts;
|
|
193
|
+
counts.truePositive += 1;
|
|
194
|
+
} else {
|
|
195
|
+
const predictedCounts = countsByLabel.get(outputLabel) as ClassCounts;
|
|
196
|
+
predictedCounts.falsePositive += 1;
|
|
197
|
+
const expectedCounts = countsByLabel.get(expectedLabel) as ClassCounts;
|
|
198
|
+
expectedCounts.falseNegative += 1;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
return countsByLabel;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Resolves the one-vs-rest positive label, if any. Auto-detection of a
|
|
206
|
+
* numeric `{0, 1}` label set only applies under the default `"macro"`
|
|
207
|
+
* average, so an explicitly configured `"micro"`/`"weighted"` average is
|
|
208
|
+
* never silently replaced by binary one-vs-rest scoring.
|
|
209
|
+
*/
|
|
210
|
+
function resolvePositiveLabel(
|
|
211
|
+
configuredPositiveLabel: ClassificationLabel | undefined,
|
|
212
|
+
labels: ClassificationLabel[],
|
|
213
|
+
average: AverageType
|
|
214
|
+
): ClassificationLabel | null {
|
|
215
|
+
if (configuredPositiveLabel !== undefined) {
|
|
216
|
+
return configuredPositiveLabel;
|
|
217
|
+
}
|
|
218
|
+
if (average !== "macro") {
|
|
219
|
+
return null;
|
|
220
|
+
}
|
|
221
|
+
const uniqueLabels = new Set(labels);
|
|
222
|
+
if (uniqueLabels.size === 2 && uniqueLabels.has(0) && uniqueLabels.has(1)) {
|
|
223
|
+
return 1;
|
|
224
|
+
}
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Aggregates per-class precision/recall/F-score across all classes.
|
|
230
|
+
*
|
|
231
|
+
* Mirrors scikit-learn's `average` semantics: for `"macro"`/`"weighted"` the
|
|
232
|
+
* F-score is computed per class and then averaged (not derived from the
|
|
233
|
+
* aggregated precision/recall). For `"micro"` the per-class confusion counts
|
|
234
|
+
* are pooled first, so the F-score is computed from the pooled
|
|
235
|
+
* precision/recall.
|
|
236
|
+
*/
|
|
237
|
+
function aggregatePrecisionRecall(
|
|
238
|
+
countsByLabel: Map<ClassificationLabel, ClassCounts>,
|
|
239
|
+
average: AverageType,
|
|
240
|
+
beta: number,
|
|
241
|
+
zeroDivision: number
|
|
242
|
+
): { precision: number; recall: number; fScore: number } {
|
|
243
|
+
if (average === "micro") {
|
|
244
|
+
let truePositive = 0;
|
|
245
|
+
let falsePositive = 0;
|
|
246
|
+
let falseNegative = 0;
|
|
247
|
+
for (const counts of countsByLabel.values()) {
|
|
248
|
+
truePositive += counts.truePositive;
|
|
249
|
+
falsePositive += counts.falsePositive;
|
|
250
|
+
falseNegative += counts.falseNegative;
|
|
251
|
+
}
|
|
252
|
+
const precision = safeDivide(
|
|
253
|
+
truePositive,
|
|
254
|
+
truePositive + falsePositive,
|
|
255
|
+
zeroDivision
|
|
256
|
+
);
|
|
257
|
+
const recall = safeDivide(
|
|
258
|
+
truePositive,
|
|
259
|
+
truePositive + falseNegative,
|
|
260
|
+
zeroDivision
|
|
261
|
+
);
|
|
262
|
+
return {
|
|
263
|
+
precision,
|
|
264
|
+
recall,
|
|
265
|
+
fScore: computeFScore(precision, recall, beta, zeroDivision),
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
const perClassMetrics = Array.from(countsByLabel.values()).map((counts) => {
|
|
270
|
+
const precision = safeDivide(
|
|
271
|
+
counts.truePositive,
|
|
272
|
+
counts.truePositive + counts.falsePositive,
|
|
273
|
+
zeroDivision
|
|
274
|
+
);
|
|
275
|
+
const recall = safeDivide(
|
|
276
|
+
counts.truePositive,
|
|
277
|
+
counts.truePositive + counts.falseNegative,
|
|
278
|
+
zeroDivision
|
|
279
|
+
);
|
|
280
|
+
return {
|
|
281
|
+
precision,
|
|
282
|
+
recall,
|
|
283
|
+
fScore: computeFScore(precision, recall, beta, zeroDivision),
|
|
284
|
+
support: counts.truePositive + counts.falseNegative,
|
|
285
|
+
};
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
if (average === "macro") {
|
|
289
|
+
return {
|
|
290
|
+
precision: mean(perClassMetrics.map((m) => m.precision)),
|
|
291
|
+
recall: mean(perClassMetrics.map((m) => m.recall)),
|
|
292
|
+
fScore: mean(perClassMetrics.map((m) => m.fScore)),
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// weighted
|
|
297
|
+
const totalSupport = perClassMetrics.reduce((sum, m) => sum + m.support, 0);
|
|
298
|
+
if (totalSupport === 0) {
|
|
299
|
+
return {
|
|
300
|
+
precision: zeroDivision,
|
|
301
|
+
recall: zeroDivision,
|
|
302
|
+
fScore: zeroDivision,
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
return {
|
|
306
|
+
precision: weightedMean(perClassMetrics, "precision", totalSupport),
|
|
307
|
+
recall: weightedMean(perClassMetrics, "recall", totalSupport),
|
|
308
|
+
fScore: weightedMean(perClassMetrics, "fScore", totalSupport),
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
function mean(values: number[]): number {
|
|
313
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
function weightedMean(
|
|
317
|
+
perClassMetrics: Array<{
|
|
318
|
+
precision: number;
|
|
319
|
+
recall: number;
|
|
320
|
+
fScore: number;
|
|
321
|
+
support: number;
|
|
322
|
+
}>,
|
|
323
|
+
field: "precision" | "recall" | "fScore",
|
|
324
|
+
totalSupport: number
|
|
325
|
+
): number {
|
|
326
|
+
const weightedSum = perClassMetrics.reduce(
|
|
327
|
+
(sum, metrics) => sum + metrics[field] * metrics.support,
|
|
328
|
+
0
|
|
329
|
+
);
|
|
330
|
+
return weightedSum / totalSupport;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* Computes precision, recall, and F-beta score for a batch of expected vs.
|
|
335
|
+
* predicted labels.
|
|
336
|
+
*
|
|
337
|
+
* `expected`/`output` are the full sequence of labels across an entire
|
|
338
|
+
* dataset, not a single row — this and the evaluators built on it are
|
|
339
|
+
* dataset-level, unlike the package's per-row LLM evaluators. Call it once
|
|
340
|
+
* over every row's collected labels rather than wiring it into a per-row
|
|
341
|
+
* pipeline (e.g. `runExperiment`'s per-row evaluators).
|
|
342
|
+
*
|
|
343
|
+
* Supports both binary classification (via `positiveLabel`, or
|
|
344
|
+
* auto-detected when `average` is at its default `"macro"` and the labels
|
|
345
|
+
* are the numeric set `{0, 1}`) and multi-class classification (via the
|
|
346
|
+
* `average` strategy).
|
|
347
|
+
*
|
|
348
|
+
* @example Multi-class (macro average)
|
|
349
|
+
* ```typescript
|
|
350
|
+
* computePrecisionRecallFScore({
|
|
351
|
+
* expected: ["cat", "dog", "cat", "bird"],
|
|
352
|
+
* output: ["cat", "cat", "cat", "bird"],
|
|
353
|
+
* });
|
|
354
|
+
* // { precision: 5/9, recall: 2/3, fScore: 0.6, beta: 1, average: "macro", ... }
|
|
355
|
+
* ```
|
|
356
|
+
*
|
|
357
|
+
* @example Binary with an explicit positive label
|
|
358
|
+
* ```typescript
|
|
359
|
+
* computePrecisionRecallFScore(
|
|
360
|
+
* { expected: ["spam", "ham", "spam"], output: ["spam", "spam", "ham"] },
|
|
361
|
+
* { beta: 0.5, positiveLabel: "spam" }
|
|
362
|
+
* );
|
|
363
|
+
* ```
|
|
364
|
+
*/
|
|
365
|
+
export function computePrecisionRecallFScore(
|
|
366
|
+
{ expected, output }: Pick<ClassificationExample, "expected" | "output">,
|
|
367
|
+
options: PrecisionRecallFScoreOptions = {}
|
|
368
|
+
): PrecisionRecallFScoreResult {
|
|
369
|
+
const {
|
|
370
|
+
beta = 1,
|
|
371
|
+
average = "macro",
|
|
372
|
+
zeroDivision = 0,
|
|
373
|
+
positiveLabel,
|
|
374
|
+
} = options;
|
|
375
|
+
|
|
376
|
+
validateBeta(beta);
|
|
377
|
+
validateAverage(average);
|
|
378
|
+
validateLabelSequences(expected, output);
|
|
379
|
+
|
|
380
|
+
const labels = collectLabels(expected, output);
|
|
381
|
+
const countsByLabel = computeClassCounts(expected, output, labels);
|
|
382
|
+
const resolvedPositiveLabel = resolvePositiveLabel(
|
|
383
|
+
positiveLabel,
|
|
384
|
+
labels,
|
|
385
|
+
average
|
|
386
|
+
);
|
|
387
|
+
|
|
388
|
+
if (resolvedPositiveLabel !== null) {
|
|
389
|
+
const classCounts = countsByLabel.get(resolvedPositiveLabel);
|
|
390
|
+
if (!classCounts) {
|
|
391
|
+
throw new Error(
|
|
392
|
+
`positiveLabel ${JSON.stringify(resolvedPositiveLabel)} not present in labels ${JSON.stringify(labels)}`
|
|
393
|
+
);
|
|
394
|
+
}
|
|
395
|
+
const precision = safeDivide(
|
|
396
|
+
classCounts.truePositive,
|
|
397
|
+
classCounts.truePositive + classCounts.falsePositive,
|
|
398
|
+
zeroDivision
|
|
399
|
+
);
|
|
400
|
+
const recall = safeDivide(
|
|
401
|
+
classCounts.truePositive,
|
|
402
|
+
classCounts.truePositive + classCounts.falseNegative,
|
|
403
|
+
zeroDivision
|
|
404
|
+
);
|
|
405
|
+
return {
|
|
406
|
+
precision,
|
|
407
|
+
recall,
|
|
408
|
+
fScore: computeFScore(precision, recall, beta, zeroDivision),
|
|
409
|
+
beta,
|
|
410
|
+
average,
|
|
411
|
+
labels,
|
|
412
|
+
positiveLabel: resolvedPositiveLabel,
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
const { precision, recall, fScore } = aggregatePrecisionRecall(
|
|
417
|
+
countsByLabel,
|
|
418
|
+
average,
|
|
419
|
+
beta,
|
|
420
|
+
zeroDivision
|
|
421
|
+
);
|
|
422
|
+
return {
|
|
423
|
+
precision,
|
|
424
|
+
recall,
|
|
425
|
+
fScore,
|
|
426
|
+
beta,
|
|
427
|
+
average,
|
|
428
|
+
labels,
|
|
429
|
+
positiveLabel: null,
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* Formats a beta value for use in metric names, e.g. `1` -> `"f1"`,
|
|
435
|
+
* `0.5` -> `"f0_5"`.
|
|
436
|
+
*/
|
|
437
|
+
export function formatBetaForMetricName(beta: number): string {
|
|
438
|
+
if (beta <= 0) {
|
|
439
|
+
return "f";
|
|
440
|
+
}
|
|
441
|
+
if (Number.isInteger(beta)) {
|
|
442
|
+
return `f${beta}`;
|
|
443
|
+
}
|
|
444
|
+
return `f${beta.toString().replace(".", "_")}`;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/**
|
|
448
|
+
* The suffix appended to a metric name to reflect the aggregation strategy,
|
|
449
|
+
* e.g. `"precision"` vs. `"precision_micro"`. No suffix is used when
|
|
450
|
+
* `positiveLabel` is explicitly configured, since `average` is not
|
|
451
|
+
* applicable in that one-vs-rest binary mode. This mirrors
|
|
452
|
+
* `resolvePositiveLabel`'s auto-detection rule (only under the default
|
|
453
|
+
* `"macro"` average) so a constructed evaluator's static name always
|
|
454
|
+
* matches what `computePrecisionRecallFScore` actually computes.
|
|
455
|
+
*/
|
|
456
|
+
export function getAverageMetricNameSuffix({
|
|
457
|
+
average = "macro",
|
|
458
|
+
positiveLabel,
|
|
459
|
+
}: PrecisionRecallFScoreOptions): string {
|
|
460
|
+
if (positiveLabel !== undefined) {
|
|
461
|
+
return "";
|
|
462
|
+
}
|
|
463
|
+
return average === "macro" ? "" : `_${average}`;
|
|
464
|
+
}
|