@arizeai/phoenix-evals 1.0.3 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/README.md +38 -0
  2. package/dist/esm/code/classificationMetrics.d.ts +121 -0
  3. package/dist/esm/code/classificationMetrics.d.ts.map +1 -0
  4. package/dist/esm/code/classificationMetrics.js +269 -0
  5. package/dist/esm/code/classificationMetrics.js.map +1 -0
  6. package/dist/esm/code/createClassificationMetricEvaluator.d.ts +16 -0
  7. package/dist/esm/code/createClassificationMetricEvaluator.d.ts.map +1 -0
  8. package/dist/esm/code/createClassificationMetricEvaluator.js +23 -0
  9. package/dist/esm/code/createClassificationMetricEvaluator.js.map +1 -0
  10. package/dist/esm/code/createF1Evaluator.d.ts +23 -0
  11. package/dist/esm/code/createF1Evaluator.d.ts.map +1 -0
  12. package/dist/esm/code/createF1Evaluator.js +23 -0
  13. package/dist/esm/code/createF1Evaluator.js.map +1 -0
  14. package/dist/esm/code/createFBetaEvaluator.d.ts +23 -0
  15. package/dist/esm/code/createFBetaEvaluator.d.ts.map +1 -0
  16. package/dist/esm/code/createFBetaEvaluator.js +27 -0
  17. package/dist/esm/code/createFBetaEvaluator.js.map +1 -0
  18. package/dist/esm/code/createPrecisionEvaluator.d.ts +22 -0
  19. package/dist/esm/code/createPrecisionEvaluator.d.ts.map +1 -0
  20. package/dist/esm/code/createPrecisionEvaluator.js +25 -0
  21. package/dist/esm/code/createPrecisionEvaluator.js.map +1 -0
  22. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
  23. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
  24. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js +44 -0
  25. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
  26. package/dist/esm/code/createRecallEvaluator.d.ts +22 -0
  27. package/dist/esm/code/createRecallEvaluator.d.ts.map +1 -0
  28. package/dist/esm/code/createRecallEvaluator.js +25 -0
  29. package/dist/esm/code/createRecallEvaluator.js.map +1 -0
  30. package/dist/esm/code/index.d.ts +7 -0
  31. package/dist/esm/code/index.d.ts.map +1 -0
  32. package/dist/esm/code/index.js +7 -0
  33. package/dist/esm/code/index.js.map +1 -0
  34. package/dist/esm/index.d.ts +1 -0
  35. package/dist/esm/index.d.ts.map +1 -1
  36. package/dist/esm/index.js +1 -0
  37. package/dist/esm/index.js.map +1 -1
  38. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  39. package/dist/src/code/classificationMetrics.d.ts +121 -0
  40. package/dist/src/code/classificationMetrics.d.ts.map +1 -0
  41. package/dist/src/code/classificationMetrics.js +274 -0
  42. package/dist/src/code/classificationMetrics.js.map +1 -0
  43. package/dist/src/code/createClassificationMetricEvaluator.d.ts +16 -0
  44. package/dist/src/code/createClassificationMetricEvaluator.d.ts.map +1 -0
  45. package/dist/src/code/createClassificationMetricEvaluator.js +26 -0
  46. package/dist/src/code/createClassificationMetricEvaluator.js.map +1 -0
  47. package/dist/src/code/createF1Evaluator.d.ts +23 -0
  48. package/dist/src/code/createF1Evaluator.d.ts.map +1 -0
  49. package/dist/src/code/createF1Evaluator.js +26 -0
  50. package/dist/src/code/createF1Evaluator.js.map +1 -0
  51. package/dist/src/code/createFBetaEvaluator.d.ts +23 -0
  52. package/dist/src/code/createFBetaEvaluator.d.ts.map +1 -0
  53. package/dist/src/code/createFBetaEvaluator.js +30 -0
  54. package/dist/src/code/createFBetaEvaluator.js.map +1 -0
  55. package/dist/src/code/createPrecisionEvaluator.d.ts +22 -0
  56. package/dist/src/code/createPrecisionEvaluator.d.ts.map +1 -0
  57. package/dist/src/code/createPrecisionEvaluator.js +28 -0
  58. package/dist/src/code/createPrecisionEvaluator.js.map +1 -0
  59. package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
  60. package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
  61. package/dist/src/code/createPrecisionRecallFScoreEvaluators.js +47 -0
  62. package/dist/src/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
  63. package/dist/src/code/createRecallEvaluator.d.ts +22 -0
  64. package/dist/src/code/createRecallEvaluator.d.ts.map +1 -0
  65. package/dist/src/code/createRecallEvaluator.js +28 -0
  66. package/dist/src/code/createRecallEvaluator.js.map +1 -0
  67. package/dist/src/code/index.d.ts +7 -0
  68. package/dist/src/code/index.d.ts.map +1 -0
  69. package/dist/src/code/index.js +23 -0
  70. package/dist/src/code/index.js.map +1 -0
  71. package/dist/src/index.d.ts +1 -0
  72. package/dist/src/index.d.ts.map +1 -1
  73. package/dist/src/index.js +1 -0
  74. package/dist/src/index.js.map +1 -1
  75. package/dist/tsconfig.tsbuildinfo +1 -1
  76. package/docs/classification-metrics.mdx +103 -0
  77. package/docs/overview.mdx +4 -1
  78. package/package.json +24 -16
  79. package/src/code/classificationMetrics.ts +464 -0
  80. package/src/code/createClassificationMetricEvaluator.ts +49 -0
  81. package/src/code/createF1Evaluator.ts +32 -0
  82. package/src/code/createFBetaEvaluator.ts +41 -0
  83. package/src/code/createPrecisionEvaluator.ts +36 -0
  84. package/src/code/createPrecisionRecallFScoreEvaluators.ts +83 -0
  85. package/src/code/createRecallEvaluator.ts +36 -0
  86. package/src/code/index.ts +6 -0
  87. package/src/index.ts +1 -0
@@ -0,0 +1,25 @@
1
+ import { getAverageMetricNameSuffix } from "./classificationMetrics.js";
2
+ import { createClassificationMetricEvaluator } from "./createClassificationMetricEvaluator.js";
3
+ /**
4
+ * Creates a code evaluator that computes precision: of the labels the model
5
+ * predicted as a given class, the fraction that were actually that class.
6
+ *
7
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
8
+ * `average` is at its default `"macro"` and labels are the numeric set
9
+ * `{0, 1}`) and multi-class classification (via the `average` strategy).
10
+ *
11
+ * @example
12
+ * ```typescript
13
+ * const precision = createPrecisionEvaluator();
14
+ * const result = await precision.evaluate({
15
+ * expected: ["cat", "dog", "cat", "bird"],
16
+ * output: ["cat", "cat", "cat", "bird"],
17
+ * });
18
+ * // { score: 5/9 }
19
+ * ```
20
+ */
21
+ export function createPrecisionEvaluator(options = {}) {
22
+ const suffix = getAverageMetricNameSuffix(options);
23
+ return createClassificationMetricEvaluator(`precision${suffix}`, "precision", options);
24
+ }
25
+ //# sourceMappingURL=createPrecisionEvaluator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createPrecisionEvaluator.js","sourceRoot":"","sources":["../../../src/code/createPrecisionEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAKrE,OAAO,EAAE,mCAAmC,EAAE,MAAM,uCAAuC,CAAC;AAE5F;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,wBAAwB,CAEtC,UAAwC,EAAE;IAC1C,MAAM,MAAM,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACnD,OAAO,mCAAmC,CACxC,YAAY,MAAM,EAAE,EACpB,WAAW,EACX,OAAO,CACR,CAAC;AACJ,CAAC"}
@@ -0,0 +1,23 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase.js";
2
+ import type { ClassificationExample, PrecisionRecallFScoreOptions } from "./classificationMetrics.js";
3
+ export interface PrecisionRecallFScoreEvaluators<RecordType extends ClassificationExample> {
4
+ precision: EvaluatorBase<RecordType>;
5
+ recall: EvaluatorBase<RecordType>;
6
+ fScore: EvaluatorBase<RecordType>;
7
+ }
8
+ /**
9
+ * Creates matching precision, recall, and F-beta evaluators from a single set
10
+ * of options, so all three are computed with the same `average`, `beta`,
11
+ * `positiveLabel`, and `zeroDivision` settings. When the same `expected`/
12
+ * `output` example object is passed to all three evaluators, the underlying
13
+ * confusion matrix is only computed once and shared across them.
14
+ *
15
+ * @example
16
+ * ```typescript
17
+ * const { precision, recall, fScore } = createPrecisionRecallFScoreEvaluators({
18
+ * average: "weighted",
19
+ * });
20
+ * ```
21
+ */
22
+ export declare function createPrecisionRecallFScoreEvaluators<RecordType extends ClassificationExample = ClassificationExample>(options?: PrecisionRecallFScoreOptions): PrecisionRecallFScoreEvaluators<RecordType>;
23
+ //# sourceMappingURL=createPrecisionRecallFScoreEvaluators.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createPrecisionRecallFScoreEvaluators.d.ts","sourceRoot":"","sources":["../../../src/code/createPrecisionRecallFScoreEvaluators.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAM3D,OAAO,KAAK,EACV,qBAAqB,EACrB,4BAA4B,EAE7B,MAAM,yBAAyB,CAAC;AAIjC,MAAM,WAAW,+BAA+B,CAC9C,UAAU,SAAS,qBAAqB;IAExC,SAAS,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IAClC,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACnC;AAoBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,qCAAqC,CACnD,UAAU,SAAS,qBAAqB,GAAG,qBAAqB,EAEhE,OAAO,GAAE,4BAAiC,GACzC,+BAA+B,CAAC,UAAU,CAAC,CAwB7C"}
@@ -0,0 +1,44 @@
1
+ import { computePrecisionRecallFScore, formatBetaForMetricName, getAverageMetricNameSuffix, } from "./classificationMetrics.js";
2
+ import { createClassificationMetricEvaluator } from "./createClassificationMetricEvaluator.js";
3
+ /**
4
+ * Wraps `computePrecisionRecallFScore` so repeated calls with the same
5
+ * `example` object (by reference) reuse the first computed result, instead
6
+ * of recomputing the full confusion matrix once per evaluator.
7
+ */
8
+ function createCachedComputer() {
9
+ const cache = new WeakMap();
10
+ return (example, options) => {
11
+ const cached = cache.get(example);
12
+ if (cached) {
13
+ return cached;
14
+ }
15
+ const result = computePrecisionRecallFScore(example, options);
16
+ cache.set(example, result);
17
+ return result;
18
+ };
19
+ }
20
+ /**
21
+ * Creates matching precision, recall, and F-beta evaluators from a single set
22
+ * of options, so all three are computed with the same `average`, `beta`,
23
+ * `positiveLabel`, and `zeroDivision` settings. When the same `expected`/
24
+ * `output` example object is passed to all three evaluators, the underlying
25
+ * confusion matrix is only computed once and shared across them.
26
+ *
27
+ * @example
28
+ * ```typescript
29
+ * const { precision, recall, fScore } = createPrecisionRecallFScoreEvaluators({
30
+ * average: "weighted",
31
+ * });
32
+ * ```
33
+ */
34
+ export function createPrecisionRecallFScoreEvaluators(options = {}) {
35
+ const { beta = 1 } = options;
36
+ const suffix = getAverageMetricNameSuffix(options);
37
+ const compute = createCachedComputer();
38
+ return {
39
+ precision: createClassificationMetricEvaluator(`precision${suffix}`, "precision", options, compute),
40
+ recall: createClassificationMetricEvaluator(`recall${suffix}`, "recall", options, compute),
41
+ fScore: createClassificationMetricEvaluator(`${formatBetaForMetricName(beta)}${suffix}`, "fScore", options, compute),
42
+ };
43
+ }
44
+ //# sourceMappingURL=createPrecisionRecallFScoreEvaluators.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createPrecisionRecallFScoreEvaluators.js","sourceRoot":"","sources":["../../../src/code/createPrecisionRecallFScoreEvaluators.ts"],"names":[],"mappings":"AACA,OAAO,EACL,4BAA4B,EAC5B,uBAAuB,EACvB,0BAA0B,GAC3B,MAAM,yBAAyB,CAAC;AAMjC,OAAO,EAAE,mCAAmC,EAAE,MAAM,uCAAuC,CAAC;AAW5F;;;;GAIG;AACH,SAAS,oBAAoB;IAC3B,MAAM,KAAK,GAAG,IAAI,OAAO,EAAuC,CAAC;IACjE,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;QAC1B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,MAAM,GAAG,4BAA4B,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC9D,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC3B,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,qCAAqC,CAGnD,UAAwC,EAAE;IAE1C,MAAM,EAAE,IAAI,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAC7B,MAAM,MAAM,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACnD,MAAM,OAAO,GAAG,oBAAoB,EAAE,CAAC;IACvC,OAAO;QACL,SAAS,EAAE,mCAAmC,CAC5C,YAAY,MAAM,EAAE,EACpB,WAAW,EACX,OAAO,EACP,OAAO,CACR;QACD,MAAM,EAAE,mCAAmC,CACzC,SAAS,MAAM,EAAE,EACjB,QAAQ,EACR,OAAO,EACP,OAAO,CACR;QACD,MAAM,EAAE,mCAAmC,CACzC,GAAG,uBAAuB,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,EAC3C,QAAQ,EACR,OAAO,EACP,OAAO,CACR;KACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,22 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase.js";
2
+ import type { ClassificationExample, PrecisionRecallFScoreOptions } from "./classificationMetrics.js";
3
+ /**
4
+ * Creates a code evaluator that computes recall: of the labels that actually
5
+ * belong to a given class, the fraction the model correctly predicted.
6
+ *
7
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
8
+ * `average` is at its default `"macro"` and labels are the numeric set
9
+ * `{0, 1}`) and multi-class classification (via the `average` strategy).
10
+ *
11
+ * @example
12
+ * ```typescript
13
+ * const recall = createRecallEvaluator();
14
+ * const result = await recall.evaluate({
15
+ * expected: ["cat", "dog", "cat", "bird"],
16
+ * output: ["cat", "cat", "cat", "bird"],
17
+ * });
18
+ * // { score: 2/3 }
19
+ * ```
20
+ */
21
+ export declare function createRecallEvaluator<RecordType extends ClassificationExample = ClassificationExample>(options?: PrecisionRecallFScoreOptions): EvaluatorBase<RecordType>;
22
+ //# sourceMappingURL=createRecallEvaluator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createRecallEvaluator.d.ts","sourceRoot":"","sources":["../../../src/code/createRecallEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,KAAK,EACV,qBAAqB,EACrB,4BAA4B,EAC7B,MAAM,yBAAyB,CAAC;AAGjC;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,SAAS,qBAAqB,GAAG,qBAAqB,EAChE,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,UAAU,CAAC,CAOvE"}
@@ -0,0 +1,25 @@
1
+ import { getAverageMetricNameSuffix } from "./classificationMetrics.js";
2
+ import { createClassificationMetricEvaluator } from "./createClassificationMetricEvaluator.js";
3
+ /**
4
+ * Creates a code evaluator that computes recall: of the labels that actually
5
+ * belong to a given class, the fraction the model correctly predicted.
6
+ *
7
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
8
+ * `average` is at its default `"macro"` and labels are the numeric set
9
+ * `{0, 1}`) and multi-class classification (via the `average` strategy).
10
+ *
11
+ * @example
12
+ * ```typescript
13
+ * const recall = createRecallEvaluator();
14
+ * const result = await recall.evaluate({
15
+ * expected: ["cat", "dog", "cat", "bird"],
16
+ * output: ["cat", "cat", "cat", "bird"],
17
+ * });
18
+ * // { score: 2/3 }
19
+ * ```
20
+ */
21
+ export function createRecallEvaluator(options = {}) {
22
+ const suffix = getAverageMetricNameSuffix(options);
23
+ return createClassificationMetricEvaluator(`recall${suffix}`, "recall", options);
24
+ }
25
+ //# sourceMappingURL=createRecallEvaluator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createRecallEvaluator.js","sourceRoot":"","sources":["../../../src/code/createRecallEvaluator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAKrE,OAAO,EAAE,mCAAmC,EAAE,MAAM,uCAAuC,CAAC;AAE5F;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,qBAAqB,CAEnC,UAAwC,EAAE;IAC1C,MAAM,MAAM,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACnD,OAAO,mCAAmC,CACxC,SAAS,MAAM,EAAE,EACjB,QAAQ,EACR,OAAO,CACR,CAAC;AACJ,CAAC"}
@@ -0,0 +1,7 @@
1
+ export * from "./classificationMetrics.js";
2
+ export * from "./createF1Evaluator.js";
3
+ export * from "./createFBetaEvaluator.js";
4
+ export * from "./createPrecisionEvaluator.js";
5
+ export * from "./createPrecisionRecallFScoreEvaluators.js";
6
+ export * from "./createRecallEvaluator.js";
7
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/code/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,yCAAyC,CAAC;AACxD,cAAc,yBAAyB,CAAC"}
@@ -0,0 +1,7 @@
1
+ export * from "./classificationMetrics.js";
2
+ export * from "./createF1Evaluator.js";
3
+ export * from "./createFBetaEvaluator.js";
4
+ export * from "./createPrecisionEvaluator.js";
5
+ export * from "./createPrecisionRecallFScoreEvaluators.js";
6
+ export * from "./createRecallEvaluator.js";
7
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/code/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,yCAAyC,CAAC;AACxD,cAAc,yBAAyB,CAAC"}
@@ -1,3 +1,4 @@
1
+ export * from "./code/index.js";
1
2
  export * from "./llm/index.js";
2
3
  export * from "./template/index.js";
3
4
  export * from "./types/index.js";
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC;AACxB,cAAc,SAAS,CAAC;AACxB,cAAc,WAAW,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,QAAQ,CAAC;AACvB,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC;AACxB,cAAc,SAAS,CAAC;AACxB,cAAc,WAAW,CAAC"}
package/dist/esm/index.js CHANGED
@@ -1,3 +1,4 @@
1
+ export * from "./code/index.js";
1
2
  export * from "./llm/index.js";
2
3
  export * from "./template/index.js";
3
4
  export * from "./types/index.js";
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC;AACxB,cAAc,SAAS,CAAC;AACxB,cAAc,WAAW,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,QAAQ,CAAC;AACvB,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC;AACxB,cAAc,SAAS,CAAC;AACxB,cAAc,WAAW,CAAC"}