@arizeai/phoenix-evals 1.0.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/README.md +39 -0
  2. package/dist/esm/code/classificationMetrics.d.ts +121 -0
  3. package/dist/esm/code/classificationMetrics.d.ts.map +1 -0
  4. package/dist/esm/code/classificationMetrics.js +269 -0
  5. package/dist/esm/code/classificationMetrics.js.map +1 -0
  6. package/dist/esm/code/createClassificationMetricEvaluator.d.ts +16 -0
  7. package/dist/esm/code/createClassificationMetricEvaluator.d.ts.map +1 -0
  8. package/dist/esm/code/createClassificationMetricEvaluator.js +23 -0
  9. package/dist/esm/code/createClassificationMetricEvaluator.js.map +1 -0
  10. package/dist/esm/code/createF1Evaluator.d.ts +23 -0
  11. package/dist/esm/code/createF1Evaluator.d.ts.map +1 -0
  12. package/dist/esm/code/createF1Evaluator.js +23 -0
  13. package/dist/esm/code/createF1Evaluator.js.map +1 -0
  14. package/dist/esm/code/createFBetaEvaluator.d.ts +23 -0
  15. package/dist/esm/code/createFBetaEvaluator.d.ts.map +1 -0
  16. package/dist/esm/code/createFBetaEvaluator.js +27 -0
  17. package/dist/esm/code/createFBetaEvaluator.js.map +1 -0
  18. package/dist/esm/code/createPrecisionEvaluator.d.ts +22 -0
  19. package/dist/esm/code/createPrecisionEvaluator.d.ts.map +1 -0
  20. package/dist/esm/code/createPrecisionEvaluator.js +25 -0
  21. package/dist/esm/code/createPrecisionEvaluator.js.map +1 -0
  22. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
  23. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
  24. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js +44 -0
  25. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
  26. package/dist/esm/code/createRecallEvaluator.d.ts +22 -0
  27. package/dist/esm/code/createRecallEvaluator.d.ts.map +1 -0
  28. package/dist/esm/code/createRecallEvaluator.js +25 -0
  29. package/dist/esm/code/createRecallEvaluator.js.map +1 -0
  30. package/dist/esm/code/index.d.ts +7 -0
  31. package/dist/esm/code/index.d.ts.map +1 -0
  32. package/dist/esm/code/index.js +7 -0
  33. package/dist/esm/code/index.js.map +1 -0
  34. package/dist/esm/index.d.ts +1 -0
  35. package/dist/esm/index.d.ts.map +1 -1
  36. package/dist/esm/index.js +1 -0
  37. package/dist/esm/index.js.map +1 -1
  38. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  39. package/dist/src/code/classificationMetrics.d.ts +121 -0
  40. package/dist/src/code/classificationMetrics.d.ts.map +1 -0
  41. package/dist/src/code/classificationMetrics.js +274 -0
  42. package/dist/src/code/classificationMetrics.js.map +1 -0
  43. package/dist/src/code/createClassificationMetricEvaluator.d.ts +16 -0
  44. package/dist/src/code/createClassificationMetricEvaluator.d.ts.map +1 -0
  45. package/dist/src/code/createClassificationMetricEvaluator.js +26 -0
  46. package/dist/src/code/createClassificationMetricEvaluator.js.map +1 -0
  47. package/dist/src/code/createF1Evaluator.d.ts +23 -0
  48. package/dist/src/code/createF1Evaluator.d.ts.map +1 -0
  49. package/dist/src/code/createF1Evaluator.js +26 -0
  50. package/dist/src/code/createF1Evaluator.js.map +1 -0
  51. package/dist/src/code/createFBetaEvaluator.d.ts +23 -0
  52. package/dist/src/code/createFBetaEvaluator.d.ts.map +1 -0
  53. package/dist/src/code/createFBetaEvaluator.js +30 -0
  54. package/dist/src/code/createFBetaEvaluator.js.map +1 -0
  55. package/dist/src/code/createPrecisionEvaluator.d.ts +22 -0
  56. package/dist/src/code/createPrecisionEvaluator.d.ts.map +1 -0
  57. package/dist/src/code/createPrecisionEvaluator.js +28 -0
  58. package/dist/src/code/createPrecisionEvaluator.js.map +1 -0
  59. package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
  60. package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
  61. package/dist/src/code/createPrecisionRecallFScoreEvaluators.js +47 -0
  62. package/dist/src/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
  63. package/dist/src/code/createRecallEvaluator.d.ts +22 -0
  64. package/dist/src/code/createRecallEvaluator.d.ts.map +1 -0
  65. package/dist/src/code/createRecallEvaluator.js +28 -0
  66. package/dist/src/code/createRecallEvaluator.js.map +1 -0
  67. package/dist/src/code/index.d.ts +7 -0
  68. package/dist/src/code/index.d.ts.map +1 -0
  69. package/dist/src/code/index.js +23 -0
  70. package/dist/src/code/index.js.map +1 -0
  71. package/dist/src/index.d.ts +1 -0
  72. package/dist/src/index.d.ts.map +1 -1
  73. package/dist/src/index.js +1 -0
  74. package/dist/src/index.js.map +1 -1
  75. package/dist/tsconfig.tsbuildinfo +1 -1
  76. package/docs/classification-metrics.mdx +103 -0
  77. package/docs/overview.mdx +4 -1
  78. package/package.json +24 -16
  79. package/src/code/classificationMetrics.ts +464 -0
  80. package/src/code/createClassificationMetricEvaluator.ts +49 -0
  81. package/src/code/createF1Evaluator.ts +32 -0
  82. package/src/code/createFBetaEvaluator.ts +41 -0
  83. package/src/code/createPrecisionEvaluator.ts +36 -0
  84. package/src/code/createPrecisionRecallFScoreEvaluators.ts +83 -0
  85. package/src/code/createRecallEvaluator.ts +36 -0
  86. package/src/code/index.ts +6 -0
  87. package/src/index.ts +1 -0
@@ -0,0 +1,103 @@
1
+ ---
2
+ title: "Classification Metrics"
3
+ description: "Precision, recall, and F-beta code evaluators in @arizeai/phoenix-evals"
4
+ ---
5
+
6
+ The `code` entrypoint provides deterministic, non-LLM evaluators for classification metrics: precision, recall, and F-beta (including F1). Unlike the per-row LLM evaluators elsewhere in this package, these are dataset-level — `expected` and `output` are the full sequence of labels across every example you want to score together, not a single row's label.
7
+
8
+ <section className="hidden" data-agent-context="relevant-source-files" aria-label="Relevant source files">
9
+ <h2>Relevant Source Files</h2>
10
+ <ul>
11
+ <li><code>src/code/index.ts</code></li>
12
+ <li><code>src/code/classificationMetrics.ts</code></li>
13
+ </ul>
14
+ </section>
15
+
16
+ ## Example
17
+
18
+ ```ts
19
+ import { createF1Evaluator } from "@arizeai/phoenix-evals/code";
20
+
21
+ const f1 = createF1Evaluator();
22
+
23
+ const result = await f1.evaluate({
24
+ expected: ["cat", "dog", "cat", "bird"],
25
+ output: ["cat", "cat", "cat", "bird"],
26
+ });
27
+ // { score: 0.6 }
28
+ ```
29
+
30
+ ## Built-In Evaluator Factories
31
+
32
+ - `createPrecisionEvaluator(options?)` — `score` is precision
33
+ - `createRecallEvaluator(options?)` — `score` is recall
34
+ - `createF1Evaluator(options?)` — `score` is F1 (F-beta with `beta: 1`)
35
+ - `createFBetaEvaluator(options?)` — `score` is F-beta; set `beta` to weight recall (`beta > 1`) or precision (`beta < 1`) more heavily
36
+ - `createPrecisionRecallFScoreEvaluators(options?)` — returns `{ precision, recall, fScore }`: three separate evaluators built from one shared options object
37
+
38
+ Each evaluator resolves to a single `{ score }`. (Python's `PrecisionRecallFScore` returns all three scores from one call; in TypeScript each evaluator yields one score, so use the bundle above to get all three.) Each factory accepts the same options:
39
+
40
+ ```ts
41
+ interface PrecisionRecallFScoreOptions {
42
+ beta?: number; // default 1
43
+ average?: "macro" | "micro" | "weighted"; // default "macro"
44
+ positiveLabel?: string | number; // for binary one-vs-rest scoring
45
+ zeroDivision?: number; // default 0, used when a metric is undefined (0/0)
46
+ }
47
+ ```
48
+
49
+ ## Binary Classification
50
+
51
+ ```ts
52
+ import {
53
+ createPrecisionEvaluator,
54
+ createRecallEvaluator,
55
+ } from "@arizeai/phoenix-evals/code";
56
+
57
+ const precision = createPrecisionEvaluator({ positiveLabel: "spam" });
58
+ const recall = createRecallEvaluator({ positiveLabel: "spam" });
59
+
60
+ const example = {
61
+ expected: ["spam", "ham", "spam", "ham", "spam"],
62
+ output: ["spam", "spam", "ham", "ham", "spam"],
63
+ };
64
+
65
+ await precision.evaluate(example); // { score: 0.667 }
66
+ await recall.evaluate(example); // { score: 0.667 }
67
+ ```
68
+
69
+ ## Multi-Class Averaging
70
+
71
+ ```ts
72
+ import { createPrecisionRecallFScoreEvaluators } from "@arizeai/phoenix-evals/code";
73
+
74
+ // "macro" (default): unweighted mean across classes
75
+ // "weighted": mean weighted by each class's support
76
+ // "micro": pool TP/FP/FN across classes first (equals accuracy for
77
+ // single-label multi-class problems)
78
+ const { precision, recall, fScore } = createPrecisionRecallFScoreEvaluators({
79
+ average: "weighted",
80
+ });
81
+ ```
82
+
83
+ ## When To Use
84
+
85
+ - Scoring classification or labeling tasks (routing, tagging, moderation categories) against ground-truth labels
86
+ - Comparing predicted vs. expected label sequences collected from an experiment or dataset
87
+ - Choosing an F-beta weighting that matches the cost of false positives vs. false negatives for your task (e.g. `beta: 2` when missing a true positive is costlier than a false alarm)
88
+
89
+ For the underlying formulas, the precision/recall tradeoff, averaging-strategy semantics, and citations, see the [Precision / Recall / F-Score](/docs/phoenix/evaluation/pre-built-metrics/precision-recall-fscore) guide. A runnable walkthrough covering binary classification, F-beta tradeoffs, and all three averaging strategies lives in the package's [`examples/classification_metrics_example.ts`](https://github.com/Arize-ai/phoenix/blob/main/js/packages/phoenix-evals/examples/classification_metrics_example.ts).
90
+
91
+ <section className="hidden" data-agent-context="source-map" aria-label="Source map">
92
+ <h2>Source Map</h2>
93
+ <ul>
94
+ <li><code>src/code/index.ts</code></li>
95
+ <li><code>src/code/classificationMetrics.ts</code></li>
96
+ <li><code>src/code/createClassificationMetricEvaluator.ts</code></li>
97
+ <li><code>src/code/createPrecisionEvaluator.ts</code></li>
98
+ <li><code>src/code/createRecallEvaluator.ts</code></li>
99
+ <li><code>src/code/createFBetaEvaluator.ts</code></li>
100
+ <li><code>src/code/createF1Evaluator.ts</code></li>
101
+ <li><code>src/code/createPrecisionRecallFScoreEvaluators.ts</code></li>
102
+ </ul>
103
+ </section>
package/docs/overview.mdx CHANGED
@@ -61,18 +61,20 @@ node_modules/@arizeai/phoenix-evals/docs/
61
61
  node_modules/@arizeai/phoenix-evals/src/
62
62
  ```
63
63
 
64
- The bundled docs cover evaluator creation, LLM evaluators, templates, classification, and Phoenix integration.
64
+ The bundled docs cover evaluator creation, LLM evaluators, classification metrics, templates, classification, and Phoenix integration.
65
65
 
66
66
  ## Where To Start
67
67
 
68
68
  - [Create evaluator](./create-evaluator) for custom and code-based evaluator flows
69
69
  - [LLM evaluators](./llm-evaluators) and [Classification](./classification) for model-backed evaluation
70
+ - [Classification metrics](./classification-metrics) for precision/recall/F-beta code evaluators
70
71
  - [Templates](./templates) and [Phoenix integration](./phoenix-integration) for prompt helpers and experiment wiring
71
72
 
72
73
  ## Source Layout
73
74
 
74
75
  - `src/index.ts` re-exports the package surface you usually import from `@arizeai/phoenix-evals`
75
76
  - `src/llm/` contains classification helpers and built-in LLM evaluator factories
77
+ - `src/code/` contains deterministic classification-metric evaluators (precision, recall, F-beta)
76
78
  - `src/helpers/` contains `createEvaluator` and evaluation-result helpers
77
79
  - `src/template/` contains `formatTemplate` and `getTemplateVariables`
78
80
  - `src/types/` contains shared evaluator and prompt types
@@ -82,6 +84,7 @@ The bundled docs cover evaluator creation, LLM evaluators, templates, classifica
82
84
  <ul>
83
85
  <li><code>src/index.ts</code></li>
84
86
  <li><code>src/llm/</code></li>
87
+ <li><code>src/code/</code></li>
85
88
  <li><code>src/helpers/</code></li>
86
89
  <li><code>src/template/</code></li>
87
90
  <li><code>src/core/</code></li>
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@arizeai/phoenix-evals",
3
- "version": "1.0.2",
3
+ "version": "1.1.0",
4
4
  "description": "A library for running evaluations for AI use cases",
5
5
  "keywords": [
6
6
  "arize",
@@ -40,6 +40,14 @@
40
40
  "import": "./dist/esm/llm/*.js",
41
41
  "require": "./dist/src/llm/*.js"
42
42
  },
43
+ "./code": {
44
+ "import": "./dist/esm/code/index.js",
45
+ "require": "./dist/src/code/index.js"
46
+ },
47
+ "./code/*": {
48
+ "import": "./dist/esm/code/*.js",
49
+ "require": "./dist/src/code/*.js"
50
+ },
43
51
  "./template/*": {
44
52
  "import": "./dist/esm/template/*.js",
45
53
  "require": "./dist/src/template/*.js"
@@ -58,25 +66,25 @@
58
66
  }
59
67
  },
60
68
  "dependencies": {
61
- "@arizeai/openinference-core": "^2.0.0",
62
- "@opentelemetry/api": "^1.9.0",
63
- "ai": "^6.0.90",
64
- "jsonpath-plus": "^10.3.0",
69
+ "@arizeai/openinference-core": "^2.4.0",
70
+ "@opentelemetry/api": "^1.9.1",
71
+ "ai": "^6.0.211",
72
+ "jsonpath-plus": "^10.4.0",
65
73
  "mustache": "^4.2.0",
66
- "zod": "^4.0.14"
74
+ "zod": "^4.4.3"
67
75
  },
68
76
  "devDependencies": {
69
- "@ai-sdk/google": "^3.0.29",
70
- "@ai-sdk/openai": "^3.0.29",
71
- "@arizeai/openinference-instrumentation-openai": "^2.3.0",
72
- "@opentelemetry/sdk-trace-node": "^1.30.1",
77
+ "@ai-sdk/google": "^3.0.84",
78
+ "@ai-sdk/openai": "^3.0.75",
79
+ "@arizeai/openinference-instrumentation-openai": "^4.1.4",
80
+ "@opentelemetry/sdk-trace-node": "^2.8.0",
73
81
  "@types/mustache": "^4.2.6",
74
- "@types/node": "^24.0.10",
75
- "msw": "^2.11.6",
76
- "nock": "^14.0.5",
77
- "tsx": "^4.19.3",
78
- "typedoc": "^0.28.17",
79
- "vitest": "^4.1.0"
82
+ "@types/node": "^25.9.4",
83
+ "msw": "^2.14.6",
84
+ "nock": "^14.0.16",
85
+ "tsx": "^4.22.4",
86
+ "typedoc": "^0.28.19",
87
+ "vitest": "^4.1.9"
80
88
  },
81
89
  "engines": {
82
90
  "node": ">=18"
@@ -0,0 +1,464 @@
1
+ /**
2
+ * A label produced by a classifier. Must be a primitive so it can be compared
3
+ * for equality and used as a lookup key.
4
+ */
5
+ export type ClassificationLabel = string | number;
6
+
7
+ /**
8
+ * The strategy used to aggregate per-class precision/recall/F-score into a
9
+ * single number when there are more than two classes (or no `positiveLabel`
10
+ * is configured).
11
+ *
12
+ * - `"macro"`: unweighted mean across classes.
13
+ * - `"micro"`: pool true/false positives and false negatives across classes
14
+ * before computing the metric.
15
+ * - `"weighted"`: mean across classes, weighted by each class's support
16
+ * (number of true instances).
17
+ */
18
+ export type AverageType = "macro" | "micro" | "weighted";
19
+
20
+ /**
21
+ * The example shape expected by the classification-metric evaluators.
22
+ */
23
+ export interface ClassificationExample {
24
+ /** The ground-truth sequence of labels. */
25
+ expected: ClassificationLabel[];
26
+ /** The predicted sequence of labels, aligned by index with `expected`. */
27
+ output: ClassificationLabel[];
28
+ [key: string]: unknown;
29
+ }
30
+
31
+ /**
32
+ * Options shared by precision, recall, and F-score computations.
33
+ */
34
+ export interface PrecisionRecallFScoreOptions {
35
+ /**
36
+ * Weight of recall relative to precision in the F-score. Must be > 0.
37
+ * @defaultValue 1
38
+ */
39
+ beta?: number;
40
+ /**
41
+ * Aggregation strategy across classes. Ignored when `positiveLabel` is set
42
+ * (or auto-detected).
43
+ * @defaultValue "macro"
44
+ */
45
+ average?: AverageType;
46
+ /**
47
+ * Value substituted for a metric when it is undefined (e.g. 0/0).
48
+ * @defaultValue 0
49
+ */
50
+ zeroDivision?: number;
51
+ /**
52
+ * When set, compute binary precision/recall/F exclusively for this label
53
+ * (one-vs-rest). If not set, `average` is at its default `"macro"`, and
54
+ * the labels are the numeric set `{0, 1}`, the positive label defaults to
55
+ * `1`. Otherwise, multi-class averaging is used. The auto-detection is
56
+ * skipped whenever a non-default `average` is configured, so an explicit
57
+ * `average` is never silently overridden by the shape of the data.
58
+ */
59
+ positiveLabel?: ClassificationLabel;
60
+ }
61
+
62
+ /**
63
+ * The result of computing precision, recall, and F-score for a batch of
64
+ * predictions.
65
+ */
66
+ export interface PrecisionRecallFScoreResult {
67
+ precision: number;
68
+ recall: number;
69
+ fScore: number;
70
+ beta: number;
71
+ average: AverageType;
72
+ /** All labels observed in `expected` and `output`, in first-seen order. */
73
+ labels: ClassificationLabel[];
74
+ /** The label treated as positive in one-vs-rest mode, or `null` if multi-class averaging was used. */
75
+ positiveLabel: ClassificationLabel | null;
76
+ }
77
+
78
+ interface ClassCounts {
79
+ truePositive: number;
80
+ falsePositive: number;
81
+ falseNegative: number;
82
+ }
83
+
84
+ function validateBeta(beta: number): void {
85
+ if (!(beta > 0)) {
86
+ throw new Error("beta must be > 0");
87
+ }
88
+ }
89
+
90
+ function validateAverage(average: AverageType): void {
91
+ if (average !== "macro" && average !== "micro" && average !== "weighted") {
92
+ throw new Error('average must be one of "macro", "micro", or "weighted"');
93
+ }
94
+ }
95
+
96
+ function validateLabelSequences(
97
+ expected: ClassificationLabel[],
98
+ output: ClassificationLabel[]
99
+ ): void {
100
+ if (!Array.isArray(expected) || !Array.isArray(output)) {
101
+ throw new Error("expected and output must be arrays of labels");
102
+ }
103
+ if (expected.length !== output.length) {
104
+ throw new Error(
105
+ `expected and output must have the same length. Got ${expected.length} and ${output.length}`
106
+ );
107
+ }
108
+ if (expected.length === 0) {
109
+ throw new Error("expected and output must be non-empty");
110
+ }
111
+ }
112
+
113
+ function safeDivide(
114
+ numerator: number,
115
+ denominator: number,
116
+ zeroDivision: number
117
+ ): number {
118
+ if (denominator === 0) {
119
+ return zeroDivision;
120
+ }
121
+ return numerator / denominator;
122
+ }
123
+
124
+ function computeFScore(
125
+ precision: number,
126
+ recall: number,
127
+ beta: number,
128
+ zeroDivision: number
129
+ ): number {
130
+ if (precision === 0 && recall === 0) {
131
+ return 0;
132
+ }
133
+ const betaSquared = beta * beta;
134
+ const numerator = (1 + betaSquared) * precision * recall;
135
+ const denominator = betaSquared * precision + recall;
136
+ return safeDivide(numerator, denominator, zeroDivision);
137
+ }
138
+
139
+ /** Collects all unique labels, ordered by first appearance in `expected` then `output`. */
140
+ function collectLabels(
141
+ expected: ClassificationLabel[],
142
+ output: ClassificationLabel[]
143
+ ): ClassificationLabel[] {
144
+ const seen = new Set<ClassificationLabel>();
145
+ const ordered: ClassificationLabel[] = [];
146
+ for (const label of [...expected, ...output]) {
147
+ if (!seen.has(label)) {
148
+ seen.add(label);
149
+ ordered.push(label);
150
+ }
151
+ }
152
+ return ordered;
153
+ }
154
+
155
+ /**
156
+ * Compares two labels the same way `Map`/`Set` key lookups do (SameValueZero),
157
+ * so a `NaN` label matches another `NaN` label instead of always mismatching
158
+ * under strict `===`.
159
+ */
160
+ function labelsAreEqual(
161
+ a: ClassificationLabel,
162
+ b: ClassificationLabel
163
+ ): boolean {
164
+ return (
165
+ a === b ||
166
+ (typeof a === "number" &&
167
+ typeof b === "number" &&
168
+ Number.isNaN(a) &&
169
+ Number.isNaN(b))
170
+ );
171
+ }
172
+
173
+ function computeClassCounts(
174
+ expected: ClassificationLabel[],
175
+ output: ClassificationLabel[],
176
+ labels: ClassificationLabel[]
177
+ ): Map<ClassificationLabel, ClassCounts> {
178
+ const countsByLabel = new Map<ClassificationLabel, ClassCounts>();
179
+ for (const label of labels) {
180
+ countsByLabel.set(label, {
181
+ truePositive: 0,
182
+ falsePositive: 0,
183
+ falseNegative: 0,
184
+ });
185
+ }
186
+ for (const [index, expectedLabel] of expected.entries()) {
187
+ // `output` is validated to have the same length as `expected`.
188
+ const outputLabel = output[index] as ClassificationLabel;
189
+ // `labels` is derived from these same `expected`/`output` arrays, so both
190
+ // labels are always pre-populated keys in `countsByLabel`.
191
+ if (labelsAreEqual(expectedLabel, outputLabel)) {
192
+ const counts = countsByLabel.get(expectedLabel) as ClassCounts;
193
+ counts.truePositive += 1;
194
+ } else {
195
+ const predictedCounts = countsByLabel.get(outputLabel) as ClassCounts;
196
+ predictedCounts.falsePositive += 1;
197
+ const expectedCounts = countsByLabel.get(expectedLabel) as ClassCounts;
198
+ expectedCounts.falseNegative += 1;
199
+ }
200
+ }
201
+ return countsByLabel;
202
+ }
203
+
204
+ /**
205
+ * Resolves the one-vs-rest positive label, if any. Auto-detection of a
206
+ * numeric `{0, 1}` label set only applies under the default `"macro"`
207
+ * average, so an explicitly configured `"micro"`/`"weighted"` average is
208
+ * never silently replaced by binary one-vs-rest scoring.
209
+ */
210
+ function resolvePositiveLabel(
211
+ configuredPositiveLabel: ClassificationLabel | undefined,
212
+ labels: ClassificationLabel[],
213
+ average: AverageType
214
+ ): ClassificationLabel | null {
215
+ if (configuredPositiveLabel !== undefined) {
216
+ return configuredPositiveLabel;
217
+ }
218
+ if (average !== "macro") {
219
+ return null;
220
+ }
221
+ const uniqueLabels = new Set(labels);
222
+ if (uniqueLabels.size === 2 && uniqueLabels.has(0) && uniqueLabels.has(1)) {
223
+ return 1;
224
+ }
225
+ return null;
226
+ }
227
+
228
+ /**
229
+ * Aggregates per-class precision/recall/F-score across all classes.
230
+ *
231
+ * Mirrors scikit-learn's `average` semantics: for `"macro"`/`"weighted"` the
232
+ * F-score is computed per class and then averaged (not derived from the
233
+ * aggregated precision/recall). For `"micro"` the per-class confusion counts
234
+ * are pooled first, so the F-score is computed from the pooled
235
+ * precision/recall.
236
+ */
237
+ function aggregatePrecisionRecall(
238
+ countsByLabel: Map<ClassificationLabel, ClassCounts>,
239
+ average: AverageType,
240
+ beta: number,
241
+ zeroDivision: number
242
+ ): { precision: number; recall: number; fScore: number } {
243
+ if (average === "micro") {
244
+ let truePositive = 0;
245
+ let falsePositive = 0;
246
+ let falseNegative = 0;
247
+ for (const counts of countsByLabel.values()) {
248
+ truePositive += counts.truePositive;
249
+ falsePositive += counts.falsePositive;
250
+ falseNegative += counts.falseNegative;
251
+ }
252
+ const precision = safeDivide(
253
+ truePositive,
254
+ truePositive + falsePositive,
255
+ zeroDivision
256
+ );
257
+ const recall = safeDivide(
258
+ truePositive,
259
+ truePositive + falseNegative,
260
+ zeroDivision
261
+ );
262
+ return {
263
+ precision,
264
+ recall,
265
+ fScore: computeFScore(precision, recall, beta, zeroDivision),
266
+ };
267
+ }
268
+
269
+ const perClassMetrics = Array.from(countsByLabel.values()).map((counts) => {
270
+ const precision = safeDivide(
271
+ counts.truePositive,
272
+ counts.truePositive + counts.falsePositive,
273
+ zeroDivision
274
+ );
275
+ const recall = safeDivide(
276
+ counts.truePositive,
277
+ counts.truePositive + counts.falseNegative,
278
+ zeroDivision
279
+ );
280
+ return {
281
+ precision,
282
+ recall,
283
+ fScore: computeFScore(precision, recall, beta, zeroDivision),
284
+ support: counts.truePositive + counts.falseNegative,
285
+ };
286
+ });
287
+
288
+ if (average === "macro") {
289
+ return {
290
+ precision: mean(perClassMetrics.map((m) => m.precision)),
291
+ recall: mean(perClassMetrics.map((m) => m.recall)),
292
+ fScore: mean(perClassMetrics.map((m) => m.fScore)),
293
+ };
294
+ }
295
+
296
+ // weighted
297
+ const totalSupport = perClassMetrics.reduce((sum, m) => sum + m.support, 0);
298
+ if (totalSupport === 0) {
299
+ return {
300
+ precision: zeroDivision,
301
+ recall: zeroDivision,
302
+ fScore: zeroDivision,
303
+ };
304
+ }
305
+ return {
306
+ precision: weightedMean(perClassMetrics, "precision", totalSupport),
307
+ recall: weightedMean(perClassMetrics, "recall", totalSupport),
308
+ fScore: weightedMean(perClassMetrics, "fScore", totalSupport),
309
+ };
310
+ }
311
+
312
+ function mean(values: number[]): number {
313
+ return values.reduce((sum, value) => sum + value, 0) / values.length;
314
+ }
315
+
316
+ function weightedMean(
317
+ perClassMetrics: Array<{
318
+ precision: number;
319
+ recall: number;
320
+ fScore: number;
321
+ support: number;
322
+ }>,
323
+ field: "precision" | "recall" | "fScore",
324
+ totalSupport: number
325
+ ): number {
326
+ const weightedSum = perClassMetrics.reduce(
327
+ (sum, metrics) => sum + metrics[field] * metrics.support,
328
+ 0
329
+ );
330
+ return weightedSum / totalSupport;
331
+ }
332
+
333
+ /**
334
+ * Computes precision, recall, and F-beta score for a batch of expected vs.
335
+ * predicted labels.
336
+ *
337
+ * `expected`/`output` are the full sequence of labels across an entire
338
+ * dataset, not a single row — this and the evaluators built on it are
339
+ * dataset-level, unlike the package's per-row LLM evaluators. Call it once
340
+ * over every row's collected labels rather than wiring it into a per-row
341
+ * pipeline (e.g. `runExperiment`'s per-row evaluators).
342
+ *
343
+ * Supports both binary classification (via `positiveLabel`, or
344
+ * auto-detected when `average` is at its default `"macro"` and the labels
345
+ * are the numeric set `{0, 1}`) and multi-class classification (via the
346
+ * `average` strategy).
347
+ *
348
+ * @example Multi-class (macro average)
349
+ * ```typescript
350
+ * computePrecisionRecallFScore({
351
+ * expected: ["cat", "dog", "cat", "bird"],
352
+ * output: ["cat", "cat", "cat", "bird"],
353
+ * });
354
+ * // { precision: 5/9, recall: 2/3, fScore: 0.6, beta: 1, average: "macro", ... }
355
+ * ```
356
+ *
357
+ * @example Binary with an explicit positive label
358
+ * ```typescript
359
+ * computePrecisionRecallFScore(
360
+ * { expected: ["spam", "ham", "spam"], output: ["spam", "spam", "ham"] },
361
+ * { beta: 0.5, positiveLabel: "spam" }
362
+ * );
363
+ * ```
364
+ */
365
+ export function computePrecisionRecallFScore(
366
+ { expected, output }: Pick<ClassificationExample, "expected" | "output">,
367
+ options: PrecisionRecallFScoreOptions = {}
368
+ ): PrecisionRecallFScoreResult {
369
+ const {
370
+ beta = 1,
371
+ average = "macro",
372
+ zeroDivision = 0,
373
+ positiveLabel,
374
+ } = options;
375
+
376
+ validateBeta(beta);
377
+ validateAverage(average);
378
+ validateLabelSequences(expected, output);
379
+
380
+ const labels = collectLabels(expected, output);
381
+ const countsByLabel = computeClassCounts(expected, output, labels);
382
+ const resolvedPositiveLabel = resolvePositiveLabel(
383
+ positiveLabel,
384
+ labels,
385
+ average
386
+ );
387
+
388
+ if (resolvedPositiveLabel !== null) {
389
+ const classCounts = countsByLabel.get(resolvedPositiveLabel);
390
+ if (!classCounts) {
391
+ throw new Error(
392
+ `positiveLabel ${JSON.stringify(resolvedPositiveLabel)} not present in labels ${JSON.stringify(labels)}`
393
+ );
394
+ }
395
+ const precision = safeDivide(
396
+ classCounts.truePositive,
397
+ classCounts.truePositive + classCounts.falsePositive,
398
+ zeroDivision
399
+ );
400
+ const recall = safeDivide(
401
+ classCounts.truePositive,
402
+ classCounts.truePositive + classCounts.falseNegative,
403
+ zeroDivision
404
+ );
405
+ return {
406
+ precision,
407
+ recall,
408
+ fScore: computeFScore(precision, recall, beta, zeroDivision),
409
+ beta,
410
+ average,
411
+ labels,
412
+ positiveLabel: resolvedPositiveLabel,
413
+ };
414
+ }
415
+
416
+ const { precision, recall, fScore } = aggregatePrecisionRecall(
417
+ countsByLabel,
418
+ average,
419
+ beta,
420
+ zeroDivision
421
+ );
422
+ return {
423
+ precision,
424
+ recall,
425
+ fScore,
426
+ beta,
427
+ average,
428
+ labels,
429
+ positiveLabel: null,
430
+ };
431
+ }
432
+
433
+ /**
434
+ * Formats a beta value for use in metric names, e.g. `1` -> `"f1"`,
435
+ * `0.5` -> `"f0_5"`.
436
+ */
437
+ export function formatBetaForMetricName(beta: number): string {
438
+ if (beta <= 0) {
439
+ return "f";
440
+ }
441
+ if (Number.isInteger(beta)) {
442
+ return `f${beta}`;
443
+ }
444
+ return `f${beta.toString().replace(".", "_")}`;
445
+ }
446
+
447
+ /**
448
+ * The suffix appended to a metric name to reflect the aggregation strategy,
449
+ * e.g. `"precision"` vs. `"precision_micro"`. No suffix is used when
450
+ * `positiveLabel` is explicitly configured, since `average` is not
451
+ * applicable in that one-vs-rest binary mode. This mirrors
452
+ * `resolvePositiveLabel`'s auto-detection rule (only under the default
453
+ * `"macro"` average) so a constructed evaluator's static name always
454
+ * matches what `computePrecisionRecallFScore` actually computes.
455
+ */
456
+ export function getAverageMetricNameSuffix({
457
+ average = "macro",
458
+ positiveLabel,
459
+ }: PrecisionRecallFScoreOptions): string {
460
+ if (positiveLabel !== undefined) {
461
+ return "";
462
+ }
463
+ return average === "macro" ? "" : `_${average}`;
464
+ }