@arizeai/phoenix-evals 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -0
- package/dist/esm/code/classificationMetrics.d.ts +121 -0
- package/dist/esm/code/classificationMetrics.d.ts.map +1 -0
- package/dist/esm/code/classificationMetrics.js +269 -0
- package/dist/esm/code/classificationMetrics.js.map +1 -0
- package/dist/esm/code/createClassificationMetricEvaluator.d.ts +16 -0
- package/dist/esm/code/createClassificationMetricEvaluator.d.ts.map +1 -0
- package/dist/esm/code/createClassificationMetricEvaluator.js +23 -0
- package/dist/esm/code/createClassificationMetricEvaluator.js.map +1 -0
- package/dist/esm/code/createF1Evaluator.d.ts +23 -0
- package/dist/esm/code/createF1Evaluator.d.ts.map +1 -0
- package/dist/esm/code/createF1Evaluator.js +23 -0
- package/dist/esm/code/createF1Evaluator.js.map +1 -0
- package/dist/esm/code/createFBetaEvaluator.d.ts +23 -0
- package/dist/esm/code/createFBetaEvaluator.d.ts.map +1 -0
- package/dist/esm/code/createFBetaEvaluator.js +27 -0
- package/dist/esm/code/createFBetaEvaluator.js.map +1 -0
- package/dist/esm/code/createPrecisionEvaluator.d.ts +22 -0
- package/dist/esm/code/createPrecisionEvaluator.d.ts.map +1 -0
- package/dist/esm/code/createPrecisionEvaluator.js +25 -0
- package/dist/esm/code/createPrecisionEvaluator.js.map +1 -0
- package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
- package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
- package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js +44 -0
- package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
- package/dist/esm/code/createRecallEvaluator.d.ts +22 -0
- package/dist/esm/code/createRecallEvaluator.d.ts.map +1 -0
- package/dist/esm/code/createRecallEvaluator.js +25 -0
- package/dist/esm/code/createRecallEvaluator.js.map +1 -0
- package/dist/esm/code/index.d.ts +7 -0
- package/dist/esm/code/index.d.ts.map +1 -0
- package/dist/esm/code/index.js +7 -0
- package/dist/esm/code/index.js.map +1 -0
- package/dist/esm/index.d.ts +1 -0
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +1 -0
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/src/code/classificationMetrics.d.ts +121 -0
- package/dist/src/code/classificationMetrics.d.ts.map +1 -0
- package/dist/src/code/classificationMetrics.js +274 -0
- package/dist/src/code/classificationMetrics.js.map +1 -0
- package/dist/src/code/createClassificationMetricEvaluator.d.ts +16 -0
- package/dist/src/code/createClassificationMetricEvaluator.d.ts.map +1 -0
- package/dist/src/code/createClassificationMetricEvaluator.js +26 -0
- package/dist/src/code/createClassificationMetricEvaluator.js.map +1 -0
- package/dist/src/code/createF1Evaluator.d.ts +23 -0
- package/dist/src/code/createF1Evaluator.d.ts.map +1 -0
- package/dist/src/code/createF1Evaluator.js +26 -0
- package/dist/src/code/createF1Evaluator.js.map +1 -0
- package/dist/src/code/createFBetaEvaluator.d.ts +23 -0
- package/dist/src/code/createFBetaEvaluator.d.ts.map +1 -0
- package/dist/src/code/createFBetaEvaluator.js +30 -0
- package/dist/src/code/createFBetaEvaluator.js.map +1 -0
- package/dist/src/code/createPrecisionEvaluator.d.ts +22 -0
- package/dist/src/code/createPrecisionEvaluator.d.ts.map +1 -0
- package/dist/src/code/createPrecisionEvaluator.js +28 -0
- package/dist/src/code/createPrecisionEvaluator.js.map +1 -0
- package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
- package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
- package/dist/src/code/createPrecisionRecallFScoreEvaluators.js +47 -0
- package/dist/src/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
- package/dist/src/code/createRecallEvaluator.d.ts +22 -0
- package/dist/src/code/createRecallEvaluator.d.ts.map +1 -0
- package/dist/src/code/createRecallEvaluator.js +28 -0
- package/dist/src/code/createRecallEvaluator.js.map +1 -0
- package/dist/src/code/index.d.ts +7 -0
- package/dist/src/code/index.d.ts.map +1 -0
- package/dist/src/code/index.js +23 -0
- package/dist/src/code/index.js.map +1 -0
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -0
- package/dist/src/index.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/docs/classification-metrics.mdx +103 -0
- package/docs/overview.mdx +4 -1
- package/package.json +24 -16
- package/src/code/classificationMetrics.ts +464 -0
- package/src/code/createClassificationMetricEvaluator.ts +49 -0
- package/src/code/createF1Evaluator.ts +32 -0
- package/src/code/createFBetaEvaluator.ts +41 -0
- package/src/code/createPrecisionEvaluator.ts +36 -0
- package/src/code/createPrecisionRecallFScoreEvaluators.ts +83 -0
- package/src/code/createRecallEvaluator.ts +36 -0
- package/src/code/index.ts +6 -0
- package/src/index.ts +1 -0
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { EvaluatorBase } from "../core/EvaluatorBase";
|
|
2
|
+
import { createEvaluator } from "../helpers/createEvaluator";
|
|
3
|
+
import {
|
|
4
|
+
computePrecisionRecallFScore,
|
|
5
|
+
type ClassificationExample,
|
|
6
|
+
type PrecisionRecallFScoreOptions,
|
|
7
|
+
type PrecisionRecallFScoreResult,
|
|
8
|
+
} from "./classificationMetrics";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Computes a {@link PrecisionRecallFScoreResult} for a batch of labels.
|
|
12
|
+
* Injectable so callers (e.g. {@link createPrecisionRecallFScoreEvaluators})
|
|
13
|
+
* can share one computed result across multiple metric evaluators instead of
|
|
14
|
+
* recomputing it per evaluator.
|
|
15
|
+
*/
|
|
16
|
+
export type ClassificationMetricComputer = (
|
|
17
|
+
example: Pick<ClassificationExample, "expected" | "output">,
|
|
18
|
+
options: PrecisionRecallFScoreOptions
|
|
19
|
+
) => PrecisionRecallFScoreResult;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Internal factory shared by `createPrecisionEvaluator`, `createRecallEvaluator`,
|
|
23
|
+
* and `createFBetaEvaluator` — each is a thin wrapper that only differs in
|
|
24
|
+
* which result field it reads and how its metric name is built.
|
|
25
|
+
*/
|
|
26
|
+
export function createClassificationMetricEvaluator<
|
|
27
|
+
RecordType extends ClassificationExample = ClassificationExample,
|
|
28
|
+
>(
|
|
29
|
+
name: string,
|
|
30
|
+
field: "precision" | "recall" | "fScore",
|
|
31
|
+
options: PrecisionRecallFScoreOptions,
|
|
32
|
+
compute: ClassificationMetricComputer = computePrecisionRecallFScore
|
|
33
|
+
): EvaluatorBase<RecordType> {
|
|
34
|
+
return createEvaluator<RecordType>(
|
|
35
|
+
// Pass `example` through by reference (rather than destructuring and
|
|
36
|
+
// rebuilding a new object) so callers that share one computed result by
|
|
37
|
+
// caching on object identity (e.g. `createPrecisionRecallFScoreEvaluators`)
|
|
38
|
+
// actually get a cache hit.
|
|
39
|
+
(example) => {
|
|
40
|
+
const result = compute(example, options);
|
|
41
|
+
return { score: result[field] };
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
name,
|
|
45
|
+
kind: "CODE",
|
|
46
|
+
optimizationDirection: "MAXIMIZE",
|
|
47
|
+
}
|
|
48
|
+
);
|
|
49
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { EvaluatorBase } from "../core/EvaluatorBase";
|
|
2
|
+
import type {
|
|
3
|
+
ClassificationExample,
|
|
4
|
+
PrecisionRecallFScoreOptions,
|
|
5
|
+
} from "./classificationMetrics";
|
|
6
|
+
import { createFBetaEvaluator } from "./createFBetaEvaluator";
|
|
7
|
+
|
|
8
|
+
export type F1EvaluatorOptions = Omit<PrecisionRecallFScoreOptions, "beta">;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Creates a code evaluator that computes the F1 score: the harmonic mean of
|
|
12
|
+
* precision and recall.
|
|
13
|
+
*
|
|
14
|
+
* Supports binary classification (via `positiveLabel`, or auto-detected when
|
|
15
|
+
* labels are the numeric set `{0, 1}`) and multi-class classification (via
|
|
16
|
+
* the `average` strategy).
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```typescript
|
|
20
|
+
* const f1 = createF1Evaluator();
|
|
21
|
+
* const result = await f1.evaluate({
|
|
22
|
+
* expected: ["cat", "dog", "cat", "bird"],
|
|
23
|
+
* output: ["cat", "cat", "cat", "bird"],
|
|
24
|
+
* });
|
|
25
|
+
* // { score: 0.6 }
|
|
26
|
+
* ```
|
|
27
|
+
*/
|
|
28
|
+
export function createF1Evaluator<
|
|
29
|
+
RecordType extends ClassificationExample = ClassificationExample,
|
|
30
|
+
>(options: F1EvaluatorOptions = {}): EvaluatorBase<RecordType> {
|
|
31
|
+
return createFBetaEvaluator<RecordType>({ ...options, beta: 1 });
|
|
32
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { EvaluatorBase } from "../core/EvaluatorBase";
|
|
2
|
+
import {
|
|
3
|
+
formatBetaForMetricName,
|
|
4
|
+
getAverageMetricNameSuffix,
|
|
5
|
+
} from "./classificationMetrics";
|
|
6
|
+
import type {
|
|
7
|
+
ClassificationExample,
|
|
8
|
+
PrecisionRecallFScoreOptions,
|
|
9
|
+
} from "./classificationMetrics";
|
|
10
|
+
import { createClassificationMetricEvaluator } from "./createClassificationMetricEvaluator";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Creates a code evaluator that computes the F-beta score: the weighted
|
|
14
|
+
* harmonic mean of precision and recall, where `beta` controls how much more
|
|
15
|
+
* weight recall gets relative to precision (`beta = 1` is the standard F1
|
|
16
|
+
* score).
|
|
17
|
+
*
|
|
18
|
+
* Supports binary classification (via `positiveLabel`, or auto-detected when
|
|
19
|
+
* `average` is at its default `"macro"` and labels are the numeric set
|
|
20
|
+
* `{0, 1}`) and multi-class classification (via the `average` strategy).
|
|
21
|
+
*
|
|
22
|
+
* @example
|
|
23
|
+
* ```typescript
|
|
24
|
+
* const f2 = createFBetaEvaluator({ beta: 2 });
|
|
25
|
+
* const result = await f2.evaluate({
|
|
26
|
+
* expected: ["cat", "dog", "cat", "bird"],
|
|
27
|
+
* output: ["cat", "cat", "cat", "bird"],
|
|
28
|
+
* });
|
|
29
|
+
* ```
|
|
30
|
+
*/
|
|
31
|
+
export function createFBetaEvaluator<
|
|
32
|
+
RecordType extends ClassificationExample = ClassificationExample,
|
|
33
|
+
>(options: PrecisionRecallFScoreOptions = {}): EvaluatorBase<RecordType> {
|
|
34
|
+
const { beta = 1 } = options;
|
|
35
|
+
const suffix = getAverageMetricNameSuffix(options);
|
|
36
|
+
return createClassificationMetricEvaluator<RecordType>(
|
|
37
|
+
`${formatBetaForMetricName(beta)}${suffix}`,
|
|
38
|
+
"fScore",
|
|
39
|
+
options
|
|
40
|
+
);
|
|
41
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { EvaluatorBase } from "../core/EvaluatorBase";
|
|
2
|
+
import { getAverageMetricNameSuffix } from "./classificationMetrics";
|
|
3
|
+
import type {
|
|
4
|
+
ClassificationExample,
|
|
5
|
+
PrecisionRecallFScoreOptions,
|
|
6
|
+
} from "./classificationMetrics";
|
|
7
|
+
import { createClassificationMetricEvaluator } from "./createClassificationMetricEvaluator";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Creates a code evaluator that computes precision: of the labels the model
|
|
11
|
+
* predicted as a given class, the fraction that were actually that class.
|
|
12
|
+
*
|
|
13
|
+
* Supports binary classification (via `positiveLabel`, or auto-detected when
|
|
14
|
+
* `average` is at its default `"macro"` and labels are the numeric set
|
|
15
|
+
* `{0, 1}`) and multi-class classification (via the `average` strategy).
|
|
16
|
+
*
|
|
17
|
+
* @example
|
|
18
|
+
* ```typescript
|
|
19
|
+
* const precision = createPrecisionEvaluator();
|
|
20
|
+
* const result = await precision.evaluate({
|
|
21
|
+
* expected: ["cat", "dog", "cat", "bird"],
|
|
22
|
+
* output: ["cat", "cat", "cat", "bird"],
|
|
23
|
+
* });
|
|
24
|
+
* // { score: 5/9 }
|
|
25
|
+
* ```
|
|
26
|
+
*/
|
|
27
|
+
export function createPrecisionEvaluator<
|
|
28
|
+
RecordType extends ClassificationExample = ClassificationExample,
|
|
29
|
+
>(options: PrecisionRecallFScoreOptions = {}): EvaluatorBase<RecordType> {
|
|
30
|
+
const suffix = getAverageMetricNameSuffix(options);
|
|
31
|
+
return createClassificationMetricEvaluator<RecordType>(
|
|
32
|
+
`precision${suffix}`,
|
|
33
|
+
"precision",
|
|
34
|
+
options
|
|
35
|
+
);
|
|
36
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import type { EvaluatorBase } from "../core/EvaluatorBase";
|
|
2
|
+
import {
|
|
3
|
+
computePrecisionRecallFScore,
|
|
4
|
+
formatBetaForMetricName,
|
|
5
|
+
getAverageMetricNameSuffix,
|
|
6
|
+
} from "./classificationMetrics";
|
|
7
|
+
import type {
|
|
8
|
+
ClassificationExample,
|
|
9
|
+
PrecisionRecallFScoreOptions,
|
|
10
|
+
PrecisionRecallFScoreResult,
|
|
11
|
+
} from "./classificationMetrics";
|
|
12
|
+
import { createClassificationMetricEvaluator } from "./createClassificationMetricEvaluator";
|
|
13
|
+
import type { ClassificationMetricComputer } from "./createClassificationMetricEvaluator";
|
|
14
|
+
|
|
15
|
+
export interface PrecisionRecallFScoreEvaluators<
|
|
16
|
+
RecordType extends ClassificationExample,
|
|
17
|
+
> {
|
|
18
|
+
precision: EvaluatorBase<RecordType>;
|
|
19
|
+
recall: EvaluatorBase<RecordType>;
|
|
20
|
+
fScore: EvaluatorBase<RecordType>;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Wraps `computePrecisionRecallFScore` so repeated calls with the same
|
|
25
|
+
* `example` object (by reference) reuse the first computed result, instead
|
|
26
|
+
* of recomputing the full confusion matrix once per evaluator.
|
|
27
|
+
*/
|
|
28
|
+
function createCachedComputer(): ClassificationMetricComputer {
|
|
29
|
+
const cache = new WeakMap<object, PrecisionRecallFScoreResult>();
|
|
30
|
+
return (example, options) => {
|
|
31
|
+
const cached = cache.get(example);
|
|
32
|
+
if (cached) {
|
|
33
|
+
return cached;
|
|
34
|
+
}
|
|
35
|
+
const result = computePrecisionRecallFScore(example, options);
|
|
36
|
+
cache.set(example, result);
|
|
37
|
+
return result;
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Creates matching precision, recall, and F-beta evaluators from a single set
|
|
43
|
+
* of options, so all three are computed with the same `average`, `beta`,
|
|
44
|
+
* `positiveLabel`, and `zeroDivision` settings. When the same `expected`/
|
|
45
|
+
* `output` example object is passed to all three evaluators, the underlying
|
|
46
|
+
* confusion matrix is only computed once and shared across them.
|
|
47
|
+
*
|
|
48
|
+
* @example
|
|
49
|
+
* ```typescript
|
|
50
|
+
* const { precision, recall, fScore } = createPrecisionRecallFScoreEvaluators({
|
|
51
|
+
* average: "weighted",
|
|
52
|
+
* });
|
|
53
|
+
* ```
|
|
54
|
+
*/
|
|
55
|
+
export function createPrecisionRecallFScoreEvaluators<
|
|
56
|
+
RecordType extends ClassificationExample = ClassificationExample,
|
|
57
|
+
>(
|
|
58
|
+
options: PrecisionRecallFScoreOptions = {}
|
|
59
|
+
): PrecisionRecallFScoreEvaluators<RecordType> {
|
|
60
|
+
const { beta = 1 } = options;
|
|
61
|
+
const suffix = getAverageMetricNameSuffix(options);
|
|
62
|
+
const compute = createCachedComputer();
|
|
63
|
+
return {
|
|
64
|
+
precision: createClassificationMetricEvaluator<RecordType>(
|
|
65
|
+
`precision${suffix}`,
|
|
66
|
+
"precision",
|
|
67
|
+
options,
|
|
68
|
+
compute
|
|
69
|
+
),
|
|
70
|
+
recall: createClassificationMetricEvaluator<RecordType>(
|
|
71
|
+
`recall${suffix}`,
|
|
72
|
+
"recall",
|
|
73
|
+
options,
|
|
74
|
+
compute
|
|
75
|
+
),
|
|
76
|
+
fScore: createClassificationMetricEvaluator<RecordType>(
|
|
77
|
+
`${formatBetaForMetricName(beta)}${suffix}`,
|
|
78
|
+
"fScore",
|
|
79
|
+
options,
|
|
80
|
+
compute
|
|
81
|
+
),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { EvaluatorBase } from "../core/EvaluatorBase";
|
|
2
|
+
import { getAverageMetricNameSuffix } from "./classificationMetrics";
|
|
3
|
+
import type {
|
|
4
|
+
ClassificationExample,
|
|
5
|
+
PrecisionRecallFScoreOptions,
|
|
6
|
+
} from "./classificationMetrics";
|
|
7
|
+
import { createClassificationMetricEvaluator } from "./createClassificationMetricEvaluator";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Creates a code evaluator that computes recall: of the labels that actually
|
|
11
|
+
* belong to a given class, the fraction the model correctly predicted.
|
|
12
|
+
*
|
|
13
|
+
* Supports binary classification (via `positiveLabel`, or auto-detected when
|
|
14
|
+
* `average` is at its default `"macro"` and labels are the numeric set
|
|
15
|
+
* `{0, 1}`) and multi-class classification (via the `average` strategy).
|
|
16
|
+
*
|
|
17
|
+
* @example
|
|
18
|
+
* ```typescript
|
|
19
|
+
* const recall = createRecallEvaluator();
|
|
20
|
+
* const result = await recall.evaluate({
|
|
21
|
+
* expected: ["cat", "dog", "cat", "bird"],
|
|
22
|
+
* output: ["cat", "cat", "cat", "bird"],
|
|
23
|
+
* });
|
|
24
|
+
* // { score: 2/3 }
|
|
25
|
+
* ```
|
|
26
|
+
*/
|
|
27
|
+
export function createRecallEvaluator<
|
|
28
|
+
RecordType extends ClassificationExample = ClassificationExample,
|
|
29
|
+
>(options: PrecisionRecallFScoreOptions = {}): EvaluatorBase<RecordType> {
|
|
30
|
+
const suffix = getAverageMetricNameSuffix(options);
|
|
31
|
+
return createClassificationMetricEvaluator<RecordType>(
|
|
32
|
+
`recall${suffix}`,
|
|
33
|
+
"recall",
|
|
34
|
+
options
|
|
35
|
+
);
|
|
36
|
+
}
|
package/src/index.ts
CHANGED