@arizeai/phoenix-evals 1.0.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/README.md +39 -0
  2. package/dist/esm/code/classificationMetrics.d.ts +121 -0
  3. package/dist/esm/code/classificationMetrics.d.ts.map +1 -0
  4. package/dist/esm/code/classificationMetrics.js +269 -0
  5. package/dist/esm/code/classificationMetrics.js.map +1 -0
  6. package/dist/esm/code/createClassificationMetricEvaluator.d.ts +16 -0
  7. package/dist/esm/code/createClassificationMetricEvaluator.d.ts.map +1 -0
  8. package/dist/esm/code/createClassificationMetricEvaluator.js +23 -0
  9. package/dist/esm/code/createClassificationMetricEvaluator.js.map +1 -0
  10. package/dist/esm/code/createF1Evaluator.d.ts +23 -0
  11. package/dist/esm/code/createF1Evaluator.d.ts.map +1 -0
  12. package/dist/esm/code/createF1Evaluator.js +23 -0
  13. package/dist/esm/code/createF1Evaluator.js.map +1 -0
  14. package/dist/esm/code/createFBetaEvaluator.d.ts +23 -0
  15. package/dist/esm/code/createFBetaEvaluator.d.ts.map +1 -0
  16. package/dist/esm/code/createFBetaEvaluator.js +27 -0
  17. package/dist/esm/code/createFBetaEvaluator.js.map +1 -0
  18. package/dist/esm/code/createPrecisionEvaluator.d.ts +22 -0
  19. package/dist/esm/code/createPrecisionEvaluator.d.ts.map +1 -0
  20. package/dist/esm/code/createPrecisionEvaluator.js +25 -0
  21. package/dist/esm/code/createPrecisionEvaluator.js.map +1 -0
  22. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
  23. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
  24. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js +44 -0
  25. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
  26. package/dist/esm/code/createRecallEvaluator.d.ts +22 -0
  27. package/dist/esm/code/createRecallEvaluator.d.ts.map +1 -0
  28. package/dist/esm/code/createRecallEvaluator.js +25 -0
  29. package/dist/esm/code/createRecallEvaluator.js.map +1 -0
  30. package/dist/esm/code/index.d.ts +7 -0
  31. package/dist/esm/code/index.d.ts.map +1 -0
  32. package/dist/esm/code/index.js +7 -0
  33. package/dist/esm/code/index.js.map +1 -0
  34. package/dist/esm/index.d.ts +1 -0
  35. package/dist/esm/index.d.ts.map +1 -1
  36. package/dist/esm/index.js +1 -0
  37. package/dist/esm/index.js.map +1 -1
  38. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  39. package/dist/src/code/classificationMetrics.d.ts +121 -0
  40. package/dist/src/code/classificationMetrics.d.ts.map +1 -0
  41. package/dist/src/code/classificationMetrics.js +274 -0
  42. package/dist/src/code/classificationMetrics.js.map +1 -0
  43. package/dist/src/code/createClassificationMetricEvaluator.d.ts +16 -0
  44. package/dist/src/code/createClassificationMetricEvaluator.d.ts.map +1 -0
  45. package/dist/src/code/createClassificationMetricEvaluator.js +26 -0
  46. package/dist/src/code/createClassificationMetricEvaluator.js.map +1 -0
  47. package/dist/src/code/createF1Evaluator.d.ts +23 -0
  48. package/dist/src/code/createF1Evaluator.d.ts.map +1 -0
  49. package/dist/src/code/createF1Evaluator.js +26 -0
  50. package/dist/src/code/createF1Evaluator.js.map +1 -0
  51. package/dist/src/code/createFBetaEvaluator.d.ts +23 -0
  52. package/dist/src/code/createFBetaEvaluator.d.ts.map +1 -0
  53. package/dist/src/code/createFBetaEvaluator.js +30 -0
  54. package/dist/src/code/createFBetaEvaluator.js.map +1 -0
  55. package/dist/src/code/createPrecisionEvaluator.d.ts +22 -0
  56. package/dist/src/code/createPrecisionEvaluator.d.ts.map +1 -0
  57. package/dist/src/code/createPrecisionEvaluator.js +28 -0
  58. package/dist/src/code/createPrecisionEvaluator.js.map +1 -0
  59. package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
  60. package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
  61. package/dist/src/code/createPrecisionRecallFScoreEvaluators.js +47 -0
  62. package/dist/src/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
  63. package/dist/src/code/createRecallEvaluator.d.ts +22 -0
  64. package/dist/src/code/createRecallEvaluator.d.ts.map +1 -0
  65. package/dist/src/code/createRecallEvaluator.js +28 -0
  66. package/dist/src/code/createRecallEvaluator.js.map +1 -0
  67. package/dist/src/code/index.d.ts +7 -0
  68. package/dist/src/code/index.d.ts.map +1 -0
  69. package/dist/src/code/index.js +23 -0
  70. package/dist/src/code/index.js.map +1 -0
  71. package/dist/src/index.d.ts +1 -0
  72. package/dist/src/index.d.ts.map +1 -1
  73. package/dist/src/index.js +1 -0
  74. package/dist/src/index.js.map +1 -1
  75. package/dist/tsconfig.tsbuildinfo +1 -1
  76. package/docs/classification-metrics.mdx +103 -0
  77. package/docs/overview.mdx +4 -1
  78. package/package.json +24 -16
  79. package/src/code/classificationMetrics.ts +464 -0
  80. package/src/code/createClassificationMetricEvaluator.ts +49 -0
  81. package/src/code/createF1Evaluator.ts +32 -0
  82. package/src/code/createFBetaEvaluator.ts +41 -0
  83. package/src/code/createPrecisionEvaluator.ts +36 -0
  84. package/src/code/createPrecisionRecallFScoreEvaluators.ts +83 -0
  85. package/src/code/createRecallEvaluator.ts +36 -0
  86. package/src/code/index.ts +6 -0
  87. package/src/index.ts +1 -0
@@ -0,0 +1,49 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase";
2
+ import { createEvaluator } from "../helpers/createEvaluator";
3
+ import {
4
+ computePrecisionRecallFScore,
5
+ type ClassificationExample,
6
+ type PrecisionRecallFScoreOptions,
7
+ type PrecisionRecallFScoreResult,
8
+ } from "./classificationMetrics";
9
+
10
+ /**
11
+ * Computes a {@link PrecisionRecallFScoreResult} for a batch of labels.
12
+ * Injectable so callers (e.g. {@link createPrecisionRecallFScoreEvaluators})
13
+ * can share one computed result across multiple metric evaluators instead of
14
+ * recomputing it per evaluator.
15
+ */
16
+ export type ClassificationMetricComputer = (
17
+ example: Pick<ClassificationExample, "expected" | "output">,
18
+ options: PrecisionRecallFScoreOptions
19
+ ) => PrecisionRecallFScoreResult;
20
+
21
+ /**
22
+ * Internal factory shared by `createPrecisionEvaluator`, `createRecallEvaluator`,
23
+ * and `createFBetaEvaluator` — each is a thin wrapper that only differs in
24
+ * which result field it reads and how its metric name is built.
25
+ */
26
+ export function createClassificationMetricEvaluator<
27
+ RecordType extends ClassificationExample = ClassificationExample,
28
+ >(
29
+ name: string,
30
+ field: "precision" | "recall" | "fScore",
31
+ options: PrecisionRecallFScoreOptions,
32
+ compute: ClassificationMetricComputer = computePrecisionRecallFScore
33
+ ): EvaluatorBase<RecordType> {
34
+ return createEvaluator<RecordType>(
35
+ // Pass `example` through by reference (rather than destructuring and
36
+ // rebuilding a new object) so callers that share one computed result by
37
+ // caching on object identity (e.g. `createPrecisionRecallFScoreEvaluators`)
38
+ // actually get a cache hit.
39
+ (example) => {
40
+ const result = compute(example, options);
41
+ return { score: result[field] };
42
+ },
43
+ {
44
+ name,
45
+ kind: "CODE",
46
+ optimizationDirection: "MAXIMIZE",
47
+ }
48
+ );
49
+ }
@@ -0,0 +1,32 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase";
2
+ import type {
3
+ ClassificationExample,
4
+ PrecisionRecallFScoreOptions,
5
+ } from "./classificationMetrics";
6
+ import { createFBetaEvaluator } from "./createFBetaEvaluator";
7
+
8
+ export type F1EvaluatorOptions = Omit<PrecisionRecallFScoreOptions, "beta">;
9
+
10
+ /**
11
+ * Creates a code evaluator that computes the F1 score: the harmonic mean of
12
+ * precision and recall.
13
+ *
14
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
15
+ * labels are the numeric set `{0, 1}`) and multi-class classification (via
16
+ * the `average` strategy).
17
+ *
18
+ * @example
19
+ * ```typescript
20
+ * const f1 = createF1Evaluator();
21
+ * const result = await f1.evaluate({
22
+ * expected: ["cat", "dog", "cat", "bird"],
23
+ * output: ["cat", "cat", "cat", "bird"],
24
+ * });
25
+ * // { score: 0.6 }
26
+ * ```
27
+ */
28
+ export function createF1Evaluator<
29
+ RecordType extends ClassificationExample = ClassificationExample,
30
+ >(options: F1EvaluatorOptions = {}): EvaluatorBase<RecordType> {
31
+ return createFBetaEvaluator<RecordType>({ ...options, beta: 1 });
32
+ }
@@ -0,0 +1,41 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase";
2
+ import {
3
+ formatBetaForMetricName,
4
+ getAverageMetricNameSuffix,
5
+ } from "./classificationMetrics";
6
+ import type {
7
+ ClassificationExample,
8
+ PrecisionRecallFScoreOptions,
9
+ } from "./classificationMetrics";
10
+ import { createClassificationMetricEvaluator } from "./createClassificationMetricEvaluator";
11
+
12
+ /**
13
+ * Creates a code evaluator that computes the F-beta score: the weighted
14
+ * harmonic mean of precision and recall, where `beta` controls how much more
15
+ * weight recall gets relative to precision (`beta = 1` is the standard F1
16
+ * score).
17
+ *
18
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
19
+ * `average` is at its default `"macro"` and labels are the numeric set
20
+ * `{0, 1}`) and multi-class classification (via the `average` strategy).
21
+ *
22
+ * @example
23
+ * ```typescript
24
+ * const f2 = createFBetaEvaluator({ beta: 2 });
25
+ * const result = await f2.evaluate({
26
+ * expected: ["cat", "dog", "cat", "bird"],
27
+ * output: ["cat", "cat", "cat", "bird"],
28
+ * });
29
+ * ```
30
+ */
31
+ export function createFBetaEvaluator<
32
+ RecordType extends ClassificationExample = ClassificationExample,
33
+ >(options: PrecisionRecallFScoreOptions = {}): EvaluatorBase<RecordType> {
34
+ const { beta = 1 } = options;
35
+ const suffix = getAverageMetricNameSuffix(options);
36
+ return createClassificationMetricEvaluator<RecordType>(
37
+ `${formatBetaForMetricName(beta)}${suffix}`,
38
+ "fScore",
39
+ options
40
+ );
41
+ }
@@ -0,0 +1,36 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase";
2
+ import { getAverageMetricNameSuffix } from "./classificationMetrics";
3
+ import type {
4
+ ClassificationExample,
5
+ PrecisionRecallFScoreOptions,
6
+ } from "./classificationMetrics";
7
+ import { createClassificationMetricEvaluator } from "./createClassificationMetricEvaluator";
8
+
9
+ /**
10
+ * Creates a code evaluator that computes precision: of the labels the model
11
+ * predicted as a given class, the fraction that were actually that class.
12
+ *
13
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
14
+ * `average` is at its default `"macro"` and labels are the numeric set
15
+ * `{0, 1}`) and multi-class classification (via the `average` strategy).
16
+ *
17
+ * @example
18
+ * ```typescript
19
+ * const precision = createPrecisionEvaluator();
20
+ * const result = await precision.evaluate({
21
+ * expected: ["cat", "dog", "cat", "bird"],
22
+ * output: ["cat", "cat", "cat", "bird"],
23
+ * });
24
+ * // { score: 5/9 }
25
+ * ```
26
+ */
27
+ export function createPrecisionEvaluator<
28
+ RecordType extends ClassificationExample = ClassificationExample,
29
+ >(options: PrecisionRecallFScoreOptions = {}): EvaluatorBase<RecordType> {
30
+ const suffix = getAverageMetricNameSuffix(options);
31
+ return createClassificationMetricEvaluator<RecordType>(
32
+ `precision${suffix}`,
33
+ "precision",
34
+ options
35
+ );
36
+ }
@@ -0,0 +1,83 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase";
2
+ import {
3
+ computePrecisionRecallFScore,
4
+ formatBetaForMetricName,
5
+ getAverageMetricNameSuffix,
6
+ } from "./classificationMetrics";
7
+ import type {
8
+ ClassificationExample,
9
+ PrecisionRecallFScoreOptions,
10
+ PrecisionRecallFScoreResult,
11
+ } from "./classificationMetrics";
12
+ import { createClassificationMetricEvaluator } from "./createClassificationMetricEvaluator";
13
+ import type { ClassificationMetricComputer } from "./createClassificationMetricEvaluator";
14
+
15
+ export interface PrecisionRecallFScoreEvaluators<
16
+ RecordType extends ClassificationExample,
17
+ > {
18
+ precision: EvaluatorBase<RecordType>;
19
+ recall: EvaluatorBase<RecordType>;
20
+ fScore: EvaluatorBase<RecordType>;
21
+ }
22
+
23
+ /**
24
+ * Wraps `computePrecisionRecallFScore` so repeated calls with the same
25
+ * `example` object (by reference) reuse the first computed result, instead
26
+ * of recomputing the full confusion matrix once per evaluator.
27
+ */
28
+ function createCachedComputer(): ClassificationMetricComputer {
29
+ const cache = new WeakMap<object, PrecisionRecallFScoreResult>();
30
+ return (example, options) => {
31
+ const cached = cache.get(example);
32
+ if (cached) {
33
+ return cached;
34
+ }
35
+ const result = computePrecisionRecallFScore(example, options);
36
+ cache.set(example, result);
37
+ return result;
38
+ };
39
+ }
40
+
41
+ /**
42
+ * Creates matching precision, recall, and F-beta evaluators from a single set
43
+ * of options, so all three are computed with the same `average`, `beta`,
44
+ * `positiveLabel`, and `zeroDivision` settings. When the same `expected`/
45
+ * `output` example object is passed to all three evaluators, the underlying
46
+ * confusion matrix is only computed once and shared across them.
47
+ *
48
+ * @example
49
+ * ```typescript
50
+ * const { precision, recall, fScore } = createPrecisionRecallFScoreEvaluators({
51
+ * average: "weighted",
52
+ * });
53
+ * ```
54
+ */
55
+ export function createPrecisionRecallFScoreEvaluators<
56
+ RecordType extends ClassificationExample = ClassificationExample,
57
+ >(
58
+ options: PrecisionRecallFScoreOptions = {}
59
+ ): PrecisionRecallFScoreEvaluators<RecordType> {
60
+ const { beta = 1 } = options;
61
+ const suffix = getAverageMetricNameSuffix(options);
62
+ const compute = createCachedComputer();
63
+ return {
64
+ precision: createClassificationMetricEvaluator<RecordType>(
65
+ `precision${suffix}`,
66
+ "precision",
67
+ options,
68
+ compute
69
+ ),
70
+ recall: createClassificationMetricEvaluator<RecordType>(
71
+ `recall${suffix}`,
72
+ "recall",
73
+ options,
74
+ compute
75
+ ),
76
+ fScore: createClassificationMetricEvaluator<RecordType>(
77
+ `${formatBetaForMetricName(beta)}${suffix}`,
78
+ "fScore",
79
+ options,
80
+ compute
81
+ ),
82
+ };
83
+ }
@@ -0,0 +1,36 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase";
2
+ import { getAverageMetricNameSuffix } from "./classificationMetrics";
3
+ import type {
4
+ ClassificationExample,
5
+ PrecisionRecallFScoreOptions,
6
+ } from "./classificationMetrics";
7
+ import { createClassificationMetricEvaluator } from "./createClassificationMetricEvaluator";
8
+
9
+ /**
10
+ * Creates a code evaluator that computes recall: of the labels that actually
11
+ * belong to a given class, the fraction the model correctly predicted.
12
+ *
13
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
14
+ * `average` is at its default `"macro"` and labels are the numeric set
15
+ * `{0, 1}`) and multi-class classification (via the `average` strategy).
16
+ *
17
+ * @example
18
+ * ```typescript
19
+ * const recall = createRecallEvaluator();
20
+ * const result = await recall.evaluate({
21
+ * expected: ["cat", "dog", "cat", "bird"],
22
+ * output: ["cat", "cat", "cat", "bird"],
23
+ * });
24
+ * // { score: 2/3 }
25
+ * ```
26
+ */
27
+ export function createRecallEvaluator<
28
+ RecordType extends ClassificationExample = ClassificationExample,
29
+ >(options: PrecisionRecallFScoreOptions = {}): EvaluatorBase<RecordType> {
30
+ const suffix = getAverageMetricNameSuffix(options);
31
+ return createClassificationMetricEvaluator<RecordType>(
32
+ `recall${suffix}`,
33
+ "recall",
34
+ options
35
+ );
36
+ }
@@ -0,0 +1,6 @@
1
+ export * from "./classificationMetrics";
2
+ export * from "./createF1Evaluator";
3
+ export * from "./createFBetaEvaluator";
4
+ export * from "./createPrecisionEvaluator";
5
+ export * from "./createPrecisionRecallFScoreEvaluators";
6
+ export * from "./createRecallEvaluator";
package/src/index.ts CHANGED
@@ -1,3 +1,4 @@
1
+ export * from "./code";
1
2
  export * from "./llm";
2
3
  export * from "./template";
3
4
  export * from "./types";