@arizeai/phoenix-evals 1.0.3 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/README.md +38 -0
  2. package/dist/esm/code/classificationMetrics.d.ts +121 -0
  3. package/dist/esm/code/classificationMetrics.d.ts.map +1 -0
  4. package/dist/esm/code/classificationMetrics.js +269 -0
  5. package/dist/esm/code/classificationMetrics.js.map +1 -0
  6. package/dist/esm/code/createClassificationMetricEvaluator.d.ts +16 -0
  7. package/dist/esm/code/createClassificationMetricEvaluator.d.ts.map +1 -0
  8. package/dist/esm/code/createClassificationMetricEvaluator.js +23 -0
  9. package/dist/esm/code/createClassificationMetricEvaluator.js.map +1 -0
  10. package/dist/esm/code/createF1Evaluator.d.ts +23 -0
  11. package/dist/esm/code/createF1Evaluator.d.ts.map +1 -0
  12. package/dist/esm/code/createF1Evaluator.js +23 -0
  13. package/dist/esm/code/createF1Evaluator.js.map +1 -0
  14. package/dist/esm/code/createFBetaEvaluator.d.ts +23 -0
  15. package/dist/esm/code/createFBetaEvaluator.d.ts.map +1 -0
  16. package/dist/esm/code/createFBetaEvaluator.js +27 -0
  17. package/dist/esm/code/createFBetaEvaluator.js.map +1 -0
  18. package/dist/esm/code/createPrecisionEvaluator.d.ts +22 -0
  19. package/dist/esm/code/createPrecisionEvaluator.d.ts.map +1 -0
  20. package/dist/esm/code/createPrecisionEvaluator.js +25 -0
  21. package/dist/esm/code/createPrecisionEvaluator.js.map +1 -0
  22. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
  23. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
  24. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js +44 -0
  25. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
  26. package/dist/esm/code/createRecallEvaluator.d.ts +22 -0
  27. package/dist/esm/code/createRecallEvaluator.d.ts.map +1 -0
  28. package/dist/esm/code/createRecallEvaluator.js +25 -0
  29. package/dist/esm/code/createRecallEvaluator.js.map +1 -0
  30. package/dist/esm/code/index.d.ts +7 -0
  31. package/dist/esm/code/index.d.ts.map +1 -0
  32. package/dist/esm/code/index.js +7 -0
  33. package/dist/esm/code/index.js.map +1 -0
  34. package/dist/esm/index.d.ts +1 -0
  35. package/dist/esm/index.d.ts.map +1 -1
  36. package/dist/esm/index.js +1 -0
  37. package/dist/esm/index.js.map +1 -1
  38. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  39. package/dist/src/code/classificationMetrics.d.ts +121 -0
  40. package/dist/src/code/classificationMetrics.d.ts.map +1 -0
  41. package/dist/src/code/classificationMetrics.js +274 -0
  42. package/dist/src/code/classificationMetrics.js.map +1 -0
  43. package/dist/src/code/createClassificationMetricEvaluator.d.ts +16 -0
  44. package/dist/src/code/createClassificationMetricEvaluator.d.ts.map +1 -0
  45. package/dist/src/code/createClassificationMetricEvaluator.js +26 -0
  46. package/dist/src/code/createClassificationMetricEvaluator.js.map +1 -0
  47. package/dist/src/code/createF1Evaluator.d.ts +23 -0
  48. package/dist/src/code/createF1Evaluator.d.ts.map +1 -0
  49. package/dist/src/code/createF1Evaluator.js +26 -0
  50. package/dist/src/code/createF1Evaluator.js.map +1 -0
  51. package/dist/src/code/createFBetaEvaluator.d.ts +23 -0
  52. package/dist/src/code/createFBetaEvaluator.d.ts.map +1 -0
  53. package/dist/src/code/createFBetaEvaluator.js +30 -0
  54. package/dist/src/code/createFBetaEvaluator.js.map +1 -0
  55. package/dist/src/code/createPrecisionEvaluator.d.ts +22 -0
  56. package/dist/src/code/createPrecisionEvaluator.d.ts.map +1 -0
  57. package/dist/src/code/createPrecisionEvaluator.js +28 -0
  58. package/dist/src/code/createPrecisionEvaluator.js.map +1 -0
  59. package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
  60. package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
  61. package/dist/src/code/createPrecisionRecallFScoreEvaluators.js +47 -0
  62. package/dist/src/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
  63. package/dist/src/code/createRecallEvaluator.d.ts +22 -0
  64. package/dist/src/code/createRecallEvaluator.d.ts.map +1 -0
  65. package/dist/src/code/createRecallEvaluator.js +28 -0
  66. package/dist/src/code/createRecallEvaluator.js.map +1 -0
  67. package/dist/src/code/index.d.ts +7 -0
  68. package/dist/src/code/index.d.ts.map +1 -0
  69. package/dist/src/code/index.js +23 -0
  70. package/dist/src/code/index.js.map +1 -0
  71. package/dist/src/index.d.ts +1 -0
  72. package/dist/src/index.d.ts.map +1 -1
  73. package/dist/src/index.js +1 -0
  74. package/dist/src/index.js.map +1 -1
  75. package/dist/tsconfig.tsbuildinfo +1 -1
  76. package/docs/classification-metrics.mdx +103 -0
  77. package/docs/overview.mdx +4 -1
  78. package/package.json +24 -16
  79. package/src/code/classificationMetrics.ts +464 -0
  80. package/src/code/createClassificationMetricEvaluator.ts +49 -0
  81. package/src/code/createF1Evaluator.ts +32 -0
  82. package/src/code/createFBetaEvaluator.ts +41 -0
  83. package/src/code/createPrecisionEvaluator.ts +36 -0
  84. package/src/code/createPrecisionRecallFScoreEvaluators.ts +83 -0
  85. package/src/code/createRecallEvaluator.ts +36 -0
  86. package/src/code/index.ts +6 -0
  87. package/src/index.ts +1 -0
package/README.md CHANGED
@@ -145,6 +145,44 @@ console.log(relevanceResult);
145
145
  // Output: { label: "relevant", score: 1, explanation: "..." }
146
146
  ```
147
147
 
148
+ ### Code Evaluators
149
+
150
+ The library also includes built-in, deterministic (non-LLM) code evaluators for common classification metrics: precision, recall, and F-beta (including F1). These are available from the `@arizeai/phoenix-evals/code` module and work over a batch of expected vs. predicted labels, supporting both binary classification (via `positiveLabel`) and multi-class classification (via `macro`/`micro`/`weighted` averaging).
151
+
152
+ - **Precision** — of everything predicted as a class, the fraction that was actually that class (`TP / (TP + FP)`). Lower precision means more false alarms.
153
+ - **Recall** — of everything that actually belongs to a class, the fraction the model found (`TP / (TP + FN)`). Lower recall means more misses.
154
+ - **F-beta** — the weighted harmonic mean of precision and recall. `beta = 1` (F1, the default) weights them equally; `beta > 1` weights recall more (use when missing a true positive is costlier, e.g. medical screening); `beta < 1` weights precision more (use when a false alarm is costlier, e.g. spam filtering).
155
+
156
+ ```typescript
157
+ import {
158
+ createPrecisionEvaluator,
159
+ createRecallEvaluator,
160
+ createF1Evaluator,
161
+ createPrecisionRecallFScoreEvaluators,
162
+ } from "@arizeai/phoenix-evals/code";
163
+
164
+ const f1 = createF1Evaluator();
165
+ const result = await f1.evaluate({
166
+ expected: ["cat", "dog", "cat", "bird"],
167
+ output: ["cat", "cat", "cat", "bird"],
168
+ });
169
+ console.log(result);
170
+ // Output: { score: 0.6 }
171
+
172
+ // Or create matching precision/recall/F-score evaluators at once
173
+ const { precision, recall, fScore } = createPrecisionRecallFScoreEvaluators({
174
+ average: "weighted",
175
+ });
176
+ ```
177
+
178
+ Use `createFBetaEvaluator({ beta })` for F-scores other than F1 (e.g. `beta: 2` weights recall higher than precision). `createPrecisionRecallFScoreEvaluators` returns three separate evaluators (`{ precision, recall, fScore }`) sharing one options object — each yields a single score, matching the individual factories.
179
+
180
+ For multi-class data, `average` controls how per-class scores combine into one number: `"macro"` (default) weights every class equally — good for surfacing whether a rare class is being ignored; `"weighted"` weights each class by how often it occurs — good when overall performance matters more than parity across classes; `"micro"` pools every true/false positive and false negative across classes first — for single-label multi-class problems this equals overall accuracy.
181
+
182
+ See [`examples/classification_metrics_example.ts`](examples/classification_metrics_example.ts) for a runnable walkthrough covering binary classification, F-beta tradeoffs, and all three averaging strategies. For the underlying formulas, the precision/recall tradeoff, and citations, see the [Precision / Recall / F-Score docs](https://arize.com/docs/phoenix/evaluation/pre-built-metrics/precision-recall-fscore).
183
+
184
+ > **Note:** unlike the per-row LLM evaluators above, these classification-metric evaluators are batch/dataset-level: `expected`/`output` are the full sequence of labels across every example, not a single row's labels. Don't wire them directly into `runExperiment` as a per-row evaluator — instead, collect every row's expected/predicted label first, then call `.evaluate({ expected, output })` once over the full arrays.
185
+
148
186
  ### Data Mapping
149
187
 
150
188
  When your data structure doesn't match what an evaluator expects, use `bindEvaluator` to map your fields to the evaluator's expected input format:
@@ -0,0 +1,121 @@
1
+ /**
2
+ * A label produced by a classifier. Must be a primitive so it can be compared
3
+ * for equality and used as a lookup key.
4
+ */
5
+ export type ClassificationLabel = string | number;
6
+ /**
7
+ * The strategy used to aggregate per-class precision/recall/F-score into a
8
+ * single number when there are more than two classes (or no `positiveLabel`
9
+ * is configured).
10
+ *
11
+ * - `"macro"`: unweighted mean across classes.
12
+ * - `"micro"`: pool true/false positives and false negatives across classes
13
+ * before computing the metric.
14
+ * - `"weighted"`: mean across classes, weighted by each class's support
15
+ * (number of true instances).
16
+ */
17
+ export type AverageType = "macro" | "micro" | "weighted";
18
+ /**
19
+ * The example shape expected by the classification-metric evaluators.
20
+ */
21
+ export interface ClassificationExample {
22
+ /** The ground-truth sequence of labels. */
23
+ expected: ClassificationLabel[];
24
+ /** The predicted sequence of labels, aligned by index with `expected`. */
25
+ output: ClassificationLabel[];
26
+ [key: string]: unknown;
27
+ }
28
+ /**
29
+ * Options shared by precision, recall, and F-score computations.
30
+ */
31
+ export interface PrecisionRecallFScoreOptions {
32
+ /**
33
+ * Weight of recall relative to precision in the F-score. Must be > 0.
34
+ * @defaultValue 1
35
+ */
36
+ beta?: number;
37
+ /**
38
+ * Aggregation strategy across classes. Ignored when `positiveLabel` is set
39
+ * (or auto-detected).
40
+ * @defaultValue "macro"
41
+ */
42
+ average?: AverageType;
43
+ /**
44
+ * Value substituted for a metric when it is undefined (e.g. 0/0).
45
+ * @defaultValue 0
46
+ */
47
+ zeroDivision?: number;
48
+ /**
49
+ * When set, compute binary precision/recall/F exclusively for this label
50
+ * (one-vs-rest). If not set, `average` is at its default `"macro"`, and
51
+ * the labels are the numeric set `{0, 1}`, the positive label defaults to
52
+ * `1`. Otherwise, multi-class averaging is used. The auto-detection is
53
+ * skipped whenever a non-default `average` is configured, so an explicit
54
+ * `average` is never silently overridden by the shape of the data.
55
+ */
56
+ positiveLabel?: ClassificationLabel;
57
+ }
58
+ /**
59
+ * The result of computing precision, recall, and F-score for a batch of
60
+ * predictions.
61
+ */
62
+ export interface PrecisionRecallFScoreResult {
63
+ precision: number;
64
+ recall: number;
65
+ fScore: number;
66
+ beta: number;
67
+ average: AverageType;
68
+ /** All labels observed in `expected` and `output`, in first-seen order. */
69
+ labels: ClassificationLabel[];
70
+ /** The label treated as positive in one-vs-rest mode, or `null` if multi-class averaging was used. */
71
+ positiveLabel: ClassificationLabel | null;
72
+ }
73
+ /**
74
+ * Computes precision, recall, and F-beta score for a batch of expected vs.
75
+ * predicted labels.
76
+ *
77
+ * `expected`/`output` are the full sequence of labels across an entire
78
+ * dataset, not a single row — this and the evaluators built on it are
79
+ * dataset-level, unlike the package's per-row LLM evaluators. Call it once
80
+ * over every row's collected labels rather than wiring it into a per-row
81
+ * pipeline (e.g. `runExperiment`'s per-row evaluators).
82
+ *
83
+ * Supports both binary classification (via `positiveLabel`, or
84
+ * auto-detected when `average` is at its default `"macro"` and the labels
85
+ * are the numeric set `{0, 1}`) and multi-class classification (via the
86
+ * `average` strategy).
87
+ *
88
+ * @example Multi-class (macro average)
89
+ * ```typescript
90
+ * computePrecisionRecallFScore({
91
+ * expected: ["cat", "dog", "cat", "bird"],
92
+ * output: ["cat", "cat", "cat", "bird"],
93
+ * });
94
+ * // { precision: 5/9, recall: 2/3, fScore: 0.6, beta: 1, average: "macro", ... }
95
+ * ```
96
+ *
97
+ * @example Binary with an explicit positive label
98
+ * ```typescript
99
+ * computePrecisionRecallFScore(
100
+ * { expected: ["spam", "ham", "spam"], output: ["spam", "spam", "ham"] },
101
+ * { beta: 0.5, positiveLabel: "spam" }
102
+ * );
103
+ * ```
104
+ */
105
+ export declare function computePrecisionRecallFScore({ expected, output }: Pick<ClassificationExample, "expected" | "output">, options?: PrecisionRecallFScoreOptions): PrecisionRecallFScoreResult;
106
+ /**
107
+ * Formats a beta value for use in metric names, e.g. `1` -> `"f1"`,
108
+ * `0.5` -> `"f0_5"`.
109
+ */
110
+ export declare function formatBetaForMetricName(beta: number): string;
111
+ /**
112
+ * The suffix appended to a metric name to reflect the aggregation strategy,
113
+ * e.g. `"precision"` vs. `"precision_micro"`. No suffix is used when
114
+ * `positiveLabel` is explicitly configured, since `average` is not
115
+ * applicable in that one-vs-rest binary mode. This mirrors
116
+ * `resolvePositiveLabel`'s auto-detection rule (only under the default
117
+ * `"macro"` average) so a constructed evaluator's static name always
118
+ * matches what `computePrecisionRecallFScore` actually computes.
119
+ */
120
+ export declare function getAverageMetricNameSuffix({ average, positiveLabel, }: PrecisionRecallFScoreOptions): string;
121
+ //# sourceMappingURL=classificationMetrics.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"classificationMetrics.d.ts","sourceRoot":"","sources":["../../../src/code/classificationMetrics.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,GAAG,MAAM,CAAC;AAElD;;;;;;;;;;GAUG;AACH,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,2CAA2C;IAC3C,QAAQ,EAAE,mBAAmB,EAAE,CAAC;IAChC,0EAA0E;IAC1E,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAC9B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,mBAAmB,CAAC;CACrC;AAED;;;GAGG;AACH,MAAM,WAAW,2BAA2B;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,WAAW,CAAC;IACrB,2EAA2E;IAC3E,MAAM,EAAE,mBAAmB,EAAE,CAAC;IAC9B,sGAAsG;IACtG,aAAa,EAAE,mBAAmB,GAAG,IAAI,CAAC;CAC3C;AAiQD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,wBAAgB,4BAA4B,CAC1C,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,qBAAqB,EAAE,UAAU,GAAG,QAAQ,CAAC,EACxE,OAAO,GAAE,4BAAiC,GACzC,2BAA2B,CA+D7B;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQ5D;AAED;;;;;;;;GAQG;AACH,wBAAgB,0BAA0B,CAAC,EACzC,OAAiB,EACjB,aAAa,GACd,EAAE,4BAA4B,GAAG,MAAM,CAKvC"}
@@ -0,0 +1,269 @@
1
+ function validateBeta(beta) {
2
+ if (!(beta > 0)) {
3
+ throw new Error("beta must be > 0");
4
+ }
5
+ }
6
+ function validateAverage(average) {
7
+ if (average !== "macro" && average !== "micro" && average !== "weighted") {
8
+ throw new Error('average must be one of "macro", "micro", or "weighted"');
9
+ }
10
+ }
11
+ function validateLabelSequences(expected, output) {
12
+ if (!Array.isArray(expected) || !Array.isArray(output)) {
13
+ throw new Error("expected and output must be arrays of labels");
14
+ }
15
+ if (expected.length !== output.length) {
16
+ throw new Error(`expected and output must have the same length. Got ${expected.length} and ${output.length}`);
17
+ }
18
+ if (expected.length === 0) {
19
+ throw new Error("expected and output must be non-empty");
20
+ }
21
+ }
22
+ function safeDivide(numerator, denominator, zeroDivision) {
23
+ if (denominator === 0) {
24
+ return zeroDivision;
25
+ }
26
+ return numerator / denominator;
27
+ }
28
+ function computeFScore(precision, recall, beta, zeroDivision) {
29
+ if (precision === 0 && recall === 0) {
30
+ return 0;
31
+ }
32
+ const betaSquared = beta * beta;
33
+ const numerator = (1 + betaSquared) * precision * recall;
34
+ const denominator = betaSquared * precision + recall;
35
+ return safeDivide(numerator, denominator, zeroDivision);
36
+ }
37
+ /** Collects all unique labels, ordered by first appearance in `expected` then `output`. */
38
+ function collectLabels(expected, output) {
39
+ const seen = new Set();
40
+ const ordered = [];
41
+ for (const label of [...expected, ...output]) {
42
+ if (!seen.has(label)) {
43
+ seen.add(label);
44
+ ordered.push(label);
45
+ }
46
+ }
47
+ return ordered;
48
+ }
49
+ /**
50
+ * Compares two labels the same way `Map`/`Set` key lookups do (SameValueZero),
51
+ * so a `NaN` label matches another `NaN` label instead of always mismatching
52
+ * under strict `===`.
53
+ */
54
+ function labelsAreEqual(a, b) {
55
+ return (a === b ||
56
+ (typeof a === "number" &&
57
+ typeof b === "number" &&
58
+ Number.isNaN(a) &&
59
+ Number.isNaN(b)));
60
+ }
61
+ function computeClassCounts(expected, output, labels) {
62
+ const countsByLabel = new Map();
63
+ for (const label of labels) {
64
+ countsByLabel.set(label, {
65
+ truePositive: 0,
66
+ falsePositive: 0,
67
+ falseNegative: 0,
68
+ });
69
+ }
70
+ for (const [index, expectedLabel] of expected.entries()) {
71
+ // `output` is validated to have the same length as `expected`.
72
+ const outputLabel = output[index];
73
+ // `labels` is derived from these same `expected`/`output` arrays, so both
74
+ // labels are always pre-populated keys in `countsByLabel`.
75
+ if (labelsAreEqual(expectedLabel, outputLabel)) {
76
+ const counts = countsByLabel.get(expectedLabel);
77
+ counts.truePositive += 1;
78
+ }
79
+ else {
80
+ const predictedCounts = countsByLabel.get(outputLabel);
81
+ predictedCounts.falsePositive += 1;
82
+ const expectedCounts = countsByLabel.get(expectedLabel);
83
+ expectedCounts.falseNegative += 1;
84
+ }
85
+ }
86
+ return countsByLabel;
87
+ }
88
+ /**
89
+ * Resolves the one-vs-rest positive label, if any. Auto-detection of a
90
+ * numeric `{0, 1}` label set only applies under the default `"macro"`
91
+ * average, so an explicitly configured `"micro"`/`"weighted"` average is
92
+ * never silently replaced by binary one-vs-rest scoring.
93
+ */
94
+ function resolvePositiveLabel(configuredPositiveLabel, labels, average) {
95
+ if (configuredPositiveLabel !== undefined) {
96
+ return configuredPositiveLabel;
97
+ }
98
+ if (average !== "macro") {
99
+ return null;
100
+ }
101
+ const uniqueLabels = new Set(labels);
102
+ if (uniqueLabels.size === 2 && uniqueLabels.has(0) && uniqueLabels.has(1)) {
103
+ return 1;
104
+ }
105
+ return null;
106
+ }
107
+ /**
108
+ * Aggregates per-class precision/recall/F-score across all classes.
109
+ *
110
+ * Mirrors scikit-learn's `average` semantics: for `"macro"`/`"weighted"` the
111
+ * F-score is computed per class and then averaged (not derived from the
112
+ * aggregated precision/recall). For `"micro"` the per-class confusion counts
113
+ * are pooled first, so the F-score is computed from the pooled
114
+ * precision/recall.
115
+ */
116
+ function aggregatePrecisionRecall(countsByLabel, average, beta, zeroDivision) {
117
+ if (average === "micro") {
118
+ let truePositive = 0;
119
+ let falsePositive = 0;
120
+ let falseNegative = 0;
121
+ for (const counts of countsByLabel.values()) {
122
+ truePositive += counts.truePositive;
123
+ falsePositive += counts.falsePositive;
124
+ falseNegative += counts.falseNegative;
125
+ }
126
+ const precision = safeDivide(truePositive, truePositive + falsePositive, zeroDivision);
127
+ const recall = safeDivide(truePositive, truePositive + falseNegative, zeroDivision);
128
+ return {
129
+ precision,
130
+ recall,
131
+ fScore: computeFScore(precision, recall, beta, zeroDivision),
132
+ };
133
+ }
134
+ const perClassMetrics = Array.from(countsByLabel.values()).map((counts) => {
135
+ const precision = safeDivide(counts.truePositive, counts.truePositive + counts.falsePositive, zeroDivision);
136
+ const recall = safeDivide(counts.truePositive, counts.truePositive + counts.falseNegative, zeroDivision);
137
+ return {
138
+ precision,
139
+ recall,
140
+ fScore: computeFScore(precision, recall, beta, zeroDivision),
141
+ support: counts.truePositive + counts.falseNegative,
142
+ };
143
+ });
144
+ if (average === "macro") {
145
+ return {
146
+ precision: mean(perClassMetrics.map((m) => m.precision)),
147
+ recall: mean(perClassMetrics.map((m) => m.recall)),
148
+ fScore: mean(perClassMetrics.map((m) => m.fScore)),
149
+ };
150
+ }
151
+ // weighted
152
+ const totalSupport = perClassMetrics.reduce((sum, m) => sum + m.support, 0);
153
+ if (totalSupport === 0) {
154
+ return {
155
+ precision: zeroDivision,
156
+ recall: zeroDivision,
157
+ fScore: zeroDivision,
158
+ };
159
+ }
160
+ return {
161
+ precision: weightedMean(perClassMetrics, "precision", totalSupport),
162
+ recall: weightedMean(perClassMetrics, "recall", totalSupport),
163
+ fScore: weightedMean(perClassMetrics, "fScore", totalSupport),
164
+ };
165
+ }
166
+ function mean(values) {
167
+ return values.reduce((sum, value) => sum + value, 0) / values.length;
168
+ }
169
+ function weightedMean(perClassMetrics, field, totalSupport) {
170
+ const weightedSum = perClassMetrics.reduce((sum, metrics) => sum + metrics[field] * metrics.support, 0);
171
+ return weightedSum / totalSupport;
172
+ }
173
+ /**
174
+ * Computes precision, recall, and F-beta score for a batch of expected vs.
175
+ * predicted labels.
176
+ *
177
+ * `expected`/`output` are the full sequence of labels across an entire
178
+ * dataset, not a single row — this and the evaluators built on it are
179
+ * dataset-level, unlike the package's per-row LLM evaluators. Call it once
180
+ * over every row's collected labels rather than wiring it into a per-row
181
+ * pipeline (e.g. `runExperiment`'s per-row evaluators).
182
+ *
183
+ * Supports both binary classification (via `positiveLabel`, or
184
+ * auto-detected when `average` is at its default `"macro"` and the labels
185
+ * are the numeric set `{0, 1}`) and multi-class classification (via the
186
+ * `average` strategy).
187
+ *
188
+ * @example Multi-class (macro average)
189
+ * ```typescript
190
+ * computePrecisionRecallFScore({
191
+ * expected: ["cat", "dog", "cat", "bird"],
192
+ * output: ["cat", "cat", "cat", "bird"],
193
+ * });
194
+ * // { precision: 5/9, recall: 2/3, fScore: 0.6, beta: 1, average: "macro", ... }
195
+ * ```
196
+ *
197
+ * @example Binary with an explicit positive label
198
+ * ```typescript
199
+ * computePrecisionRecallFScore(
200
+ * { expected: ["spam", "ham", "spam"], output: ["spam", "spam", "ham"] },
201
+ * { beta: 0.5, positiveLabel: "spam" }
202
+ * );
203
+ * ```
204
+ */
205
+ export function computePrecisionRecallFScore({ expected, output }, options = {}) {
206
+ const { beta = 1, average = "macro", zeroDivision = 0, positiveLabel, } = options;
207
+ validateBeta(beta);
208
+ validateAverage(average);
209
+ validateLabelSequences(expected, output);
210
+ const labels = collectLabels(expected, output);
211
+ const countsByLabel = computeClassCounts(expected, output, labels);
212
+ const resolvedPositiveLabel = resolvePositiveLabel(positiveLabel, labels, average);
213
+ if (resolvedPositiveLabel !== null) {
214
+ const classCounts = countsByLabel.get(resolvedPositiveLabel);
215
+ if (!classCounts) {
216
+ throw new Error(`positiveLabel ${JSON.stringify(resolvedPositiveLabel)} not present in labels ${JSON.stringify(labels)}`);
217
+ }
218
+ const precision = safeDivide(classCounts.truePositive, classCounts.truePositive + classCounts.falsePositive, zeroDivision);
219
+ const recall = safeDivide(classCounts.truePositive, classCounts.truePositive + classCounts.falseNegative, zeroDivision);
220
+ return {
221
+ precision,
222
+ recall,
223
+ fScore: computeFScore(precision, recall, beta, zeroDivision),
224
+ beta,
225
+ average,
226
+ labels,
227
+ positiveLabel: resolvedPositiveLabel,
228
+ };
229
+ }
230
+ const { precision, recall, fScore } = aggregatePrecisionRecall(countsByLabel, average, beta, zeroDivision);
231
+ return {
232
+ precision,
233
+ recall,
234
+ fScore,
235
+ beta,
236
+ average,
237
+ labels,
238
+ positiveLabel: null,
239
+ };
240
+ }
241
+ /**
242
+ * Formats a beta value for use in metric names, e.g. `1` -> `"f1"`,
243
+ * `0.5` -> `"f0_5"`.
244
+ */
245
+ export function formatBetaForMetricName(beta) {
246
+ if (beta <= 0) {
247
+ return "f";
248
+ }
249
+ if (Number.isInteger(beta)) {
250
+ return `f${beta}`;
251
+ }
252
+ return `f${beta.toString().replace(".", "_")}`;
253
+ }
254
+ /**
255
+ * The suffix appended to a metric name to reflect the aggregation strategy,
256
+ * e.g. `"precision"` vs. `"precision_micro"`. No suffix is used when
257
+ * `positiveLabel` is explicitly configured, since `average` is not
258
+ * applicable in that one-vs-rest binary mode. This mirrors
259
+ * `resolvePositiveLabel`'s auto-detection rule (only under the default
260
+ * `"macro"` average) so a constructed evaluator's static name always
261
+ * matches what `computePrecisionRecallFScore` actually computes.
262
+ */
263
+ export function getAverageMetricNameSuffix({ average = "macro", positiveLabel, }) {
264
+ if (positiveLabel !== undefined) {
265
+ return "";
266
+ }
267
+ return average === "macro" ? "" : `_${average}`;
268
+ }
269
+ //# sourceMappingURL=classificationMetrics.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"classificationMetrics.js","sourceRoot":"","sources":["../../../src/code/classificationMetrics.ts"],"names":[],"mappings":"AAmFA,SAAS,YAAY,CAAC,IAAY;IAChC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACtC,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,OAAoB;IAC3C,IAAI,OAAO,KAAK,OAAO,IAAI,OAAO,KAAK,OAAO,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC5E,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAC7B,QAA+B,EAC/B,MAA6B;IAE7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACb,sDAAsD,QAAQ,CAAC,MAAM,QAAQ,MAAM,CAAC,MAAM,EAAE,CAC7F,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CACjB,SAAiB,EACjB,WAAmB,EACnB,YAAoB;IAEpB,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,OAAO,SAAS,GAAG,WAAW,CAAC;AACjC,CAAC;AAED,SAAS,aAAa,CACpB,SAAiB,EACjB,MAAc,EACd,IAAY,EACZ,YAAoB;IAEpB,IAAI,SAAS,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,GAAG,IAAI,CAAC;IAChC,MAAM,SAAS,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC;IACzD,MAAM,WAAW,GAAG,WAAW,GAAG,SAAS,GAAG,MAAM,CAAC;IACrD,OAAO,UAAU,CAAC,SAAS,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;AAC1D,CAAC;AAED,2FAA2F;AAC3F,SAAS,aAAa,CACpB,QAA+B,EAC/B,MAA6B;IAE7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,MAAM,OAAO,GAA0B,EAAE,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CACrB,CAAsB,EACtB,CAAsB;IAEtB,OAAO,CACL,CAAC,KAAK,CAAC;QACP,CAAC,OAAO,CAAC,KAAK,QAAQ;YACpB,OAAO,CAAC,KAAK,QAAQ;YACrB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CACnB,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CACzB,QAA+B,EAC/B,MAA6B,EAC7B,MAA6B;IAE7B,MAAM,aAAa,GAAG,IAAI,GAAG,EAAoC,CAAC;IAClE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE;YACvB,YAAY,EAAE,CAAC;YACf,aAAa,EAAE,CAAC;YAChB,aAAa,EAAE,CAAC;SACjB,CAAC,CAAC;IACL,CAAC;IACD,KAAK,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACxD,+DAA+D;QAC/D,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAwB,CAAC;QACzD,0EAA0E;QAC1E,2DAA2D;QAC3D,IAAI,cAAc,CAAC,aAAa,EAAE,WAAW,CAAC,EAAE,CAAC;YAC/C,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,aAAa,CAAgB,CAAC;YAC/D,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,MAAM,eAAe,GAAG,aAAa,CAAC,GAAG,CAAC,WAAW,CAAgB,CAAC;YACtE,eAAe,CAAC,aAAa,IAAI,CAAC,CAAC;YACnC,MAAM,cAAc,GAAG,aAAa,CAAC,GAAG,CAAC,aAAa,CAAgB,CAAC;YACvE,cAAc,CAAC,aAAa,IAAI,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;;;GAKG;AACH,SAAS,oBAAoB,CAC3B,uBAAwD,EACxD,MAA6B,EAC7B,OAAoB;IAEpB,IAAI,uBAAuB,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,uBAAuB,CAAC;IACjC,CAAC;IACD,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACrC,IAAI,YAAY,CAAC,IAAI,KAAK,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,wBAAwB,CAC/B,aAAoD,EACpD,OAAoB,EACpB,IAAY,EACZ,YAAoB;IAEpB,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACxB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,KAAK,MAAM,MAAM,IAAI,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC;YACpC,aAAa,IAAI,MAAM,CAAC,aAAa,CAAC;YACtC,aAAa,IAAI,MAAM,CAAC,aAAa,CAAC;QACxC,CAAC;QACD,MAAM,SAAS,GAAG,UAAU,CAC1B,YAAY,EACZ,YAAY,GAAG,aAAa,EAC5B,YAAY,CACb,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,CACvB,YAAY,EACZ,YAAY,GAAG,aAAa,EAC5B,YAAY,CACb,CAAC;QACF,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC;SAC7D,CAAC;IACJ,CAAC;IAED,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACxE,MAAM,SAAS,GAAG,UAAU,CAC1B,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,aAAa,EAC1C,YAAY,CACb,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,CACvB,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,aAAa,EAC1C,YAAY,CACb,CAAC;QACF,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC;YAC5D,OAAO,EAAE,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,aAAa;SACpD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;QACxB,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACxD,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAClD,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;SACnD,CAAC;IACJ,CAAC;IAED,WAAW;IACX,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC5E,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO;YACL,SAAS,EAAE,YAAY;YACvB,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE,YAAY;SACrB,CAAC;IACJ,CAAC;IACD,OAAO;QACL,SAAS,EAAE,YAAY,CAAC,eAAe,EAAE,WAAW,EAAE,YAAY,CAAC;QACnE,MAAM,EAAE,YAAY,CAAC,eAAe,EAAE,QAAQ,EAAE,YAAY,CAAC;QAC7D,MAAM,EAAE,YAAY,CAAC,eAAe,EAAE,QAAQ,EAAE,YAAY,CAAC;KAC9D,CAAC;AACJ,CAAC;AAED,SAAS,IAAI,CAAC,MAAgB;IAC5B,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACvE,CAAC;AAED,SAAS,YAAY,CACnB,eAKE,EACF,KAAwC,EACxC,YAAoB;IAEpB,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CACxC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,EACxD,CAAC,CACF,CAAC;IACF,OAAO,WAAW,GAAG,YAAY,CAAC;AACpC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,UAAU,4BAA4B,CAC1C,EAAE,QAAQ,EAAE,MAAM,EAAsD,EACxE,UAAwC,EAAE;IAE1C,MAAM,EACJ,IAAI,GAAG,CAAC,EACR,OAAO,GAAG,OAAO,EACjB,YAAY,GAAG,CAAC,EAChB,aAAa,GACd,GAAG,OAAO,CAAC;IAEZ,YAAY,CAAC,IAAI,CAAC,CAAC;IACnB,eAAe,CAAC,OAAO,CAAC,CAAC;IACzB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEzC,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,aAAa,GAAG,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACnE,MAAM,qBAAqB,GAAG,oBAAoB,CAChD,aAAa,EACb,MAAM,EACN,OAAO,CACR,CAAC;IAEF,IAAI,qBAAqB,KAAK,IAAI,EAAE,CAAC;QACnC,MAAM,WAAW,GAAG,aAAa,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QAC7D,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACb,iBAAiB,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CACzG,CAAC;QACJ,CAAC;QACD,MAAM,SAAS,GAAG,UAAU,CAC1B,WAAW,CAAC,YAAY,EACxB,WAAW,CAAC,YAAY,GAAG,WAAW,CAAC,aAAa,EACpD,YAAY,CACb,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,CACvB,WAAW,CAAC,YAAY,EACxB,WAAW,CAAC,YAAY,GAAG,WAAW,CAAC,aAAa,EACpD,YAAY,CACb,CAAC;QACF,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC;YAC5D,IAAI;YACJ,OAAO;YACP,MAAM;YACN,aAAa,EAAE,qBAAqB;SACrC,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,wBAAwB,CAC5D,aAAa,EACb,OAAO,EACP,IAAI,EACJ,YAAY,CACb,CAAC;IACF,OAAO;QACL,SAAS;QACT,MAAM;QACN,MAAM;QACN,IAAI;QACJ,OAAO;QACP,MAAM;QACN,aAAa,EAAE,IAAI;KACpB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAY;IAClD,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QACd,OAAO,GAAG,CAAC;IACb,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,OAAO,IAAI,IAAI,EAAE,CAAC;IACpB,CAAC;IACD,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;AACjD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,0BAA0B,CAAC,EACzC,OAAO,GAAG,OAAO,EACjB,aAAa,GACgB;IAC7B,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAChC,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,OAAO,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;AAClD,CAAC"}
@@ -0,0 +1,16 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase.js";
2
+ import { type ClassificationExample, type PrecisionRecallFScoreOptions, type PrecisionRecallFScoreResult } from "./classificationMetrics.js";
3
+ /**
4
+ * Computes a {@link PrecisionRecallFScoreResult} for a batch of labels.
5
+ * Injectable so callers (e.g. {@link createPrecisionRecallFScoreEvaluators})
6
+ * can share one computed result across multiple metric evaluators instead of
7
+ * recomputing it per evaluator.
8
+ */
9
+ export type ClassificationMetricComputer = (example: Pick<ClassificationExample, "expected" | "output">, options: PrecisionRecallFScoreOptions) => PrecisionRecallFScoreResult;
10
+ /**
11
+ * Internal factory shared by `createPrecisionEvaluator`, `createRecallEvaluator`,
12
+ * and `createFBetaEvaluator` — each is a thin wrapper that only differs in
13
+ * which result field it reads and how its metric name is built.
14
+ */
15
+ export declare function createClassificationMetricEvaluator<RecordType extends ClassificationExample = ClassificationExample>(name: string, field: "precision" | "recall" | "fScore", options: PrecisionRecallFScoreOptions, compute?: ClassificationMetricComputer): EvaluatorBase<RecordType>;
16
+ //# sourceMappingURL=createClassificationMetricEvaluator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createClassificationMetricEvaluator.d.ts","sourceRoot":"","sources":["../../../src/code/createClassificationMetricEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,EAEL,KAAK,qBAAqB,EAC1B,KAAK,4BAA4B,EACjC,KAAK,2BAA2B,EACjC,MAAM,yBAAyB,CAAC;AAEjC;;;;;GAKG;AACH,MAAM,MAAM,4BAA4B,GAAG,CACzC,OAAO,EAAE,IAAI,CAAC,qBAAqB,EAAE,UAAU,GAAG,QAAQ,CAAC,EAC3D,OAAO,EAAE,4BAA4B,KAClC,2BAA2B,CAAC;AAEjC;;;;GAIG;AACH,wBAAgB,mCAAmC,CACjD,UAAU,SAAS,qBAAqB,GAAG,qBAAqB,EAEhE,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,WAAW,GAAG,QAAQ,GAAG,QAAQ,EACxC,OAAO,EAAE,4BAA4B,EACrC,OAAO,GAAE,4BAA2D,GACnE,aAAa,CAAC,UAAU,CAAC,CAgB3B"}
@@ -0,0 +1,23 @@
1
+ import { createEvaluator } from "../helpers/createEvaluator.js";
2
+ import { computePrecisionRecallFScore, } from "./classificationMetrics.js";
3
+ /**
4
+ * Internal factory shared by `createPrecisionEvaluator`, `createRecallEvaluator`,
5
+ * and `createFBetaEvaluator` — each is a thin wrapper that only differs in
6
+ * which result field it reads and how its metric name is built.
7
+ */
8
+ export function createClassificationMetricEvaluator(name, field, options, compute = computePrecisionRecallFScore) {
9
+ return createEvaluator(
10
+ // Pass `example` through by reference (rather than destructuring and
11
+ // rebuilding a new object) so callers that share one computed result by
12
+ // caching on object identity (e.g. `createPrecisionRecallFScoreEvaluators`)
13
+ // actually get a cache hit.
14
+ (example) => {
15
+ const result = compute(example, options);
16
+ return { score: result[field] };
17
+ }, {
18
+ name,
19
+ kind: "CODE",
20
+ optimizationDirection: "MAXIMIZE",
21
+ });
22
+ }
23
+ //# sourceMappingURL=createClassificationMetricEvaluator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createClassificationMetricEvaluator.js","sourceRoot":"","sources":["../../../src/code/createClassificationMetricEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EACL,4BAA4B,GAI7B,MAAM,yBAAyB,CAAC;AAajC;;;;GAIG;AACH,MAAM,UAAU,mCAAmC,CAGjD,IAAY,EACZ,KAAwC,EACxC,OAAqC,EACrC,UAAwC,4BAA4B;IAEpE,OAAO,eAAe;IACpB,qEAAqE;IACrE,wEAAwE;IACxE,4EAA4E;IAC5E,4BAA4B;IAC5B,CAAC,OAAO,EAAE,EAAE;QACV,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IAClC,CAAC,EACD;QACE,IAAI;QACJ,IAAI,EAAE,MAAM;QACZ,qBAAqB,EAAE,UAAU;KAClC,CACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,23 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase.js";
2
+ import type { ClassificationExample, PrecisionRecallFScoreOptions } from "./classificationMetrics.js";
3
+ export type F1EvaluatorOptions = Omit<PrecisionRecallFScoreOptions, "beta">;
4
+ /**
5
+ * Creates a code evaluator that computes the F1 score: the harmonic mean of
6
+ * precision and recall.
7
+ *
8
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
9
+ * labels are the numeric set `{0, 1}`) and multi-class classification (via
10
+ * the `average` strategy).
11
+ *
12
+ * @example
13
+ * ```typescript
14
+ * const f1 = createF1Evaluator();
15
+ * const result = await f1.evaluate({
16
+ * expected: ["cat", "dog", "cat", "bird"],
17
+ * output: ["cat", "cat", "cat", "bird"],
18
+ * });
19
+ * // { score: 0.6 }
20
+ * ```
21
+ */
22
+ export declare function createF1Evaluator<RecordType extends ClassificationExample = ClassificationExample>(options?: F1EvaluatorOptions): EvaluatorBase<RecordType>;
23
+ //# sourceMappingURL=createF1Evaluator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createF1Evaluator.d.ts","sourceRoot":"","sources":["../../../src/code/createF1Evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EACV,qBAAqB,EACrB,4BAA4B,EAC7B,MAAM,yBAAyB,CAAC;AAGjC,MAAM,MAAM,kBAAkB,GAAG,IAAI,CAAC,4BAA4B,EAAE,MAAM,CAAC,CAAC;AAE5E;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,iBAAiB,CAC/B,UAAU,SAAS,qBAAqB,GAAG,qBAAqB,EAChE,OAAO,GAAE,kBAAuB,GAAG,aAAa,CAAC,UAAU,CAAC,CAE7D"}
@@ -0,0 +1,23 @@
1
+ import { createFBetaEvaluator } from "./createFBetaEvaluator.js";
2
+ /**
3
+ * Creates a code evaluator that computes the F1 score: the harmonic mean of
4
+ * precision and recall.
5
+ *
6
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
7
+ * labels are the numeric set `{0, 1}`) and multi-class classification (via
8
+ * the `average` strategy).
9
+ *
10
+ * @example
11
+ * ```typescript
12
+ * const f1 = createF1Evaluator();
13
+ * const result = await f1.evaluate({
14
+ * expected: ["cat", "dog", "cat", "bird"],
15
+ * output: ["cat", "cat", "cat", "bird"],
16
+ * });
17
+ * // { score: 0.6 }
18
+ * ```
19
+ */
20
+ export function createF1Evaluator(options = {}) {
21
+ return createFBetaEvaluator({ ...options, beta: 1 });
22
+ }
23
+ //# sourceMappingURL=createF1Evaluator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createF1Evaluator.js","sourceRoot":"","sources":["../../../src/code/createF1Evaluator.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAI9D;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,iBAAiB,CAE/B,UAA8B,EAAE;IAChC,OAAO,oBAAoB,CAAa,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;AACnE,CAAC"}
@@ -0,0 +1,23 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase.js";
2
+ import type { ClassificationExample, PrecisionRecallFScoreOptions } from "./classificationMetrics.js";
3
+ /**
4
+ * Creates a code evaluator that computes the F-beta score: the weighted
5
+ * harmonic mean of precision and recall, where `beta` controls how much more
6
+ * weight recall gets relative to precision (`beta = 1` is the standard F1
7
+ * score).
8
+ *
9
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
10
+ * `average` is at its default `"macro"` and labels are the numeric set
11
+ * `{0, 1}`) and multi-class classification (via the `average` strategy).
12
+ *
13
+ * @example
14
+ * ```typescript
15
+ * const f2 = createFBetaEvaluator({ beta: 2 });
16
+ * const result = await f2.evaluate({
17
+ * expected: ["cat", "dog", "cat", "bird"],
18
+ * output: ["cat", "cat", "cat", "bird"],
19
+ * });
20
+ * ```
21
+ */
22
+ export declare function createFBetaEvaluator<RecordType extends ClassificationExample = ClassificationExample>(options?: PrecisionRecallFScoreOptions): EvaluatorBase<RecordType>;
23
+ //# sourceMappingURL=createFBetaEvaluator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createFBetaEvaluator.d.ts","sourceRoot":"","sources":["../../../src/code/createFBetaEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAK3D,OAAO,KAAK,EACV,qBAAqB,EACrB,4BAA4B,EAC7B,MAAM,yBAAyB,CAAC;AAGjC;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,oBAAoB,CAClC,UAAU,SAAS,qBAAqB,GAAG,qBAAqB,EAChE,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,UAAU,CAAC,CAQvE"}
@@ -0,0 +1,27 @@
1
+ import { formatBetaForMetricName, getAverageMetricNameSuffix, } from "./classificationMetrics.js";
2
+ import { createClassificationMetricEvaluator } from "./createClassificationMetricEvaluator.js";
3
+ /**
4
+ * Creates a code evaluator that computes the F-beta score: the weighted
5
+ * harmonic mean of precision and recall, where `beta` controls how much more
6
+ * weight recall gets relative to precision (`beta = 1` is the standard F1
7
+ * score).
8
+ *
9
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
10
+ * `average` is at its default `"macro"` and labels are the numeric set
11
+ * `{0, 1}`) and multi-class classification (via the `average` strategy).
12
+ *
13
+ * @example
14
+ * ```typescript
15
+ * const f2 = createFBetaEvaluator({ beta: 2 });
16
+ * const result = await f2.evaluate({
17
+ * expected: ["cat", "dog", "cat", "bird"],
18
+ * output: ["cat", "cat", "cat", "bird"],
19
+ * });
20
+ * ```
21
+ */
22
+ export function createFBetaEvaluator(options = {}) {
23
+ const { beta = 1 } = options;
24
+ const suffix = getAverageMetricNameSuffix(options);
25
+ return createClassificationMetricEvaluator(`${formatBetaForMetricName(beta)}${suffix}`, "fScore", options);
26
+ }
27
+ //# sourceMappingURL=createFBetaEvaluator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createFBetaEvaluator.js","sourceRoot":"","sources":["../../../src/code/createFBetaEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EACL,uBAAuB,EACvB,0BAA0B,GAC3B,MAAM,yBAAyB,CAAC;AAKjC,OAAO,EAAE,mCAAmC,EAAE,MAAM,uCAAuC,CAAC;AAE5F;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,oBAAoB,CAElC,UAAwC,EAAE;IAC1C,MAAM,EAAE,IAAI,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAC7B,MAAM,MAAM,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACnD,OAAO,mCAAmC,CACxC,GAAG,uBAAuB,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,EAC3C,QAAQ,EACR,OAAO,CACR,CAAC;AACJ,CAAC"}
@@ -0,0 +1,22 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase.js";
2
+ import type { ClassificationExample, PrecisionRecallFScoreOptions } from "./classificationMetrics.js";
3
+ /**
4
+ * Creates a code evaluator that computes precision: of the labels the model
5
+ * predicted as a given class, the fraction that were actually that class.
6
+ *
7
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
8
+ * `average` is at its default `"macro"` and labels are the numeric set
9
+ * `{0, 1}`) and multi-class classification (via the `average` strategy).
10
+ *
11
+ * @example
12
+ * ```typescript
13
+ * const precision = createPrecisionEvaluator();
14
+ * const result = await precision.evaluate({
15
+ * expected: ["cat", "dog", "cat", "bird"],
16
+ * output: ["cat", "cat", "cat", "bird"],
17
+ * });
18
+ * // { score: 5/9 }
19
+ * ```
20
+ */
21
+ export declare function createPrecisionEvaluator<RecordType extends ClassificationExample = ClassificationExample>(options?: PrecisionRecallFScoreOptions): EvaluatorBase<RecordType>;
22
+ //# sourceMappingURL=createPrecisionEvaluator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createPrecisionEvaluator.d.ts","sourceRoot":"","sources":["../../../src/code/createPrecisionEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,KAAK,EACV,qBAAqB,EACrB,4BAA4B,EAC7B,MAAM,yBAAyB,CAAC;AAGjC;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,wBAAwB,CACtC,UAAU,SAAS,qBAAqB,GAAG,qBAAqB,EAChE,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,UAAU,CAAC,CAOvE"}