@arizeai/phoenix-evals 1.0.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/README.md +39 -0
  2. package/dist/esm/code/classificationMetrics.d.ts +121 -0
  3. package/dist/esm/code/classificationMetrics.d.ts.map +1 -0
  4. package/dist/esm/code/classificationMetrics.js +269 -0
  5. package/dist/esm/code/classificationMetrics.js.map +1 -0
  6. package/dist/esm/code/createClassificationMetricEvaluator.d.ts +16 -0
  7. package/dist/esm/code/createClassificationMetricEvaluator.d.ts.map +1 -0
  8. package/dist/esm/code/createClassificationMetricEvaluator.js +23 -0
  9. package/dist/esm/code/createClassificationMetricEvaluator.js.map +1 -0
  10. package/dist/esm/code/createF1Evaluator.d.ts +23 -0
  11. package/dist/esm/code/createF1Evaluator.d.ts.map +1 -0
  12. package/dist/esm/code/createF1Evaluator.js +23 -0
  13. package/dist/esm/code/createF1Evaluator.js.map +1 -0
  14. package/dist/esm/code/createFBetaEvaluator.d.ts +23 -0
  15. package/dist/esm/code/createFBetaEvaluator.d.ts.map +1 -0
  16. package/dist/esm/code/createFBetaEvaluator.js +27 -0
  17. package/dist/esm/code/createFBetaEvaluator.js.map +1 -0
  18. package/dist/esm/code/createPrecisionEvaluator.d.ts +22 -0
  19. package/dist/esm/code/createPrecisionEvaluator.d.ts.map +1 -0
  20. package/dist/esm/code/createPrecisionEvaluator.js +25 -0
  21. package/dist/esm/code/createPrecisionEvaluator.js.map +1 -0
  22. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
  23. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
  24. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js +44 -0
  25. package/dist/esm/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
  26. package/dist/esm/code/createRecallEvaluator.d.ts +22 -0
  27. package/dist/esm/code/createRecallEvaluator.d.ts.map +1 -0
  28. package/dist/esm/code/createRecallEvaluator.js +25 -0
  29. package/dist/esm/code/createRecallEvaluator.js.map +1 -0
  30. package/dist/esm/code/index.d.ts +7 -0
  31. package/dist/esm/code/index.d.ts.map +1 -0
  32. package/dist/esm/code/index.js +7 -0
  33. package/dist/esm/code/index.js.map +1 -0
  34. package/dist/esm/index.d.ts +1 -0
  35. package/dist/esm/index.d.ts.map +1 -1
  36. package/dist/esm/index.js +1 -0
  37. package/dist/esm/index.js.map +1 -1
  38. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  39. package/dist/src/code/classificationMetrics.d.ts +121 -0
  40. package/dist/src/code/classificationMetrics.d.ts.map +1 -0
  41. package/dist/src/code/classificationMetrics.js +274 -0
  42. package/dist/src/code/classificationMetrics.js.map +1 -0
  43. package/dist/src/code/createClassificationMetricEvaluator.d.ts +16 -0
  44. package/dist/src/code/createClassificationMetricEvaluator.d.ts.map +1 -0
  45. package/dist/src/code/createClassificationMetricEvaluator.js +26 -0
  46. package/dist/src/code/createClassificationMetricEvaluator.js.map +1 -0
  47. package/dist/src/code/createF1Evaluator.d.ts +23 -0
  48. package/dist/src/code/createF1Evaluator.d.ts.map +1 -0
  49. package/dist/src/code/createF1Evaluator.js +26 -0
  50. package/dist/src/code/createF1Evaluator.js.map +1 -0
  51. package/dist/src/code/createFBetaEvaluator.d.ts +23 -0
  52. package/dist/src/code/createFBetaEvaluator.d.ts.map +1 -0
  53. package/dist/src/code/createFBetaEvaluator.js +30 -0
  54. package/dist/src/code/createFBetaEvaluator.js.map +1 -0
  55. package/dist/src/code/createPrecisionEvaluator.d.ts +22 -0
  56. package/dist/src/code/createPrecisionEvaluator.d.ts.map +1 -0
  57. package/dist/src/code/createPrecisionEvaluator.js +28 -0
  58. package/dist/src/code/createPrecisionEvaluator.js.map +1 -0
  59. package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts +23 -0
  60. package/dist/src/code/createPrecisionRecallFScoreEvaluators.d.ts.map +1 -0
  61. package/dist/src/code/createPrecisionRecallFScoreEvaluators.js +47 -0
  62. package/dist/src/code/createPrecisionRecallFScoreEvaluators.js.map +1 -0
  63. package/dist/src/code/createRecallEvaluator.d.ts +22 -0
  64. package/dist/src/code/createRecallEvaluator.d.ts.map +1 -0
  65. package/dist/src/code/createRecallEvaluator.js +28 -0
  66. package/dist/src/code/createRecallEvaluator.js.map +1 -0
  67. package/dist/src/code/index.d.ts +7 -0
  68. package/dist/src/code/index.d.ts.map +1 -0
  69. package/dist/src/code/index.js +23 -0
  70. package/dist/src/code/index.js.map +1 -0
  71. package/dist/src/index.d.ts +1 -0
  72. package/dist/src/index.d.ts.map +1 -1
  73. package/dist/src/index.js +1 -0
  74. package/dist/src/index.js.map +1 -1
  75. package/dist/tsconfig.tsbuildinfo +1 -1
  76. package/docs/classification-metrics.mdx +103 -0
  77. package/docs/overview.mdx +4 -1
  78. package/package.json +24 -16
  79. package/src/code/classificationMetrics.ts +464 -0
  80. package/src/code/createClassificationMetricEvaluator.ts +49 -0
  81. package/src/code/createF1Evaluator.ts +32 -0
  82. package/src/code/createFBetaEvaluator.ts +41 -0
  83. package/src/code/createPrecisionEvaluator.ts +36 -0
  84. package/src/code/createPrecisionRecallFScoreEvaluators.ts +83 -0
  85. package/src/code/createRecallEvaluator.ts +36 -0
  86. package/src/code/index.ts +6 -0
  87. package/src/index.ts +1 -0
@@ -0,0 +1,23 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase";
2
+ import type { ClassificationExample, PrecisionRecallFScoreOptions } from "./classificationMetrics";
3
+ export interface PrecisionRecallFScoreEvaluators<RecordType extends ClassificationExample> {
4
+ precision: EvaluatorBase<RecordType>;
5
+ recall: EvaluatorBase<RecordType>;
6
+ fScore: EvaluatorBase<RecordType>;
7
+ }
8
+ /**
9
+ * Creates matching precision, recall, and F-beta evaluators from a single set
10
+ * of options, so all three are computed with the same `average`, `beta`,
11
+ * `positiveLabel`, and `zeroDivision` settings. When the same `expected`/
12
+ * `output` example object is passed to all three evaluators, the underlying
13
+ * confusion matrix is only computed once and shared across them.
14
+ *
15
+ * @example
16
+ * ```typescript
17
+ * const { precision, recall, fScore } = createPrecisionRecallFScoreEvaluators({
18
+ * average: "weighted",
19
+ * });
20
+ * ```
21
+ */
22
+ export declare function createPrecisionRecallFScoreEvaluators<RecordType extends ClassificationExample = ClassificationExample>(options?: PrecisionRecallFScoreOptions): PrecisionRecallFScoreEvaluators<RecordType>;
23
+ //# sourceMappingURL=createPrecisionRecallFScoreEvaluators.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createPrecisionRecallFScoreEvaluators.d.ts","sourceRoot":"","sources":["../../../src/code/createPrecisionRecallFScoreEvaluators.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAM3D,OAAO,KAAK,EACV,qBAAqB,EACrB,4BAA4B,EAE7B,MAAM,yBAAyB,CAAC;AAIjC,MAAM,WAAW,+BAA+B,CAC9C,UAAU,SAAS,qBAAqB;IAExC,SAAS,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IAClC,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACnC;AAoBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,qCAAqC,CACnD,UAAU,SAAS,qBAAqB,GAAG,qBAAqB,EAEhE,OAAO,GAAE,4BAAiC,GACzC,+BAA+B,CAAC,UAAU,CAAC,CAwB7C"}
@@ -0,0 +1,47 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createPrecisionRecallFScoreEvaluators = createPrecisionRecallFScoreEvaluators;
4
+ const classificationMetrics_1 = require("./classificationMetrics");
5
+ const createClassificationMetricEvaluator_1 = require("./createClassificationMetricEvaluator");
6
+ /**
7
+ * Wraps `computePrecisionRecallFScore` so repeated calls with the same
8
+ * `example` object (by reference) reuse the first computed result, instead
9
+ * of recomputing the full confusion matrix once per evaluator.
10
+ */
11
+ function createCachedComputer() {
12
+ const cache = new WeakMap();
13
+ return (example, options) => {
14
+ const cached = cache.get(example);
15
+ if (cached) {
16
+ return cached;
17
+ }
18
+ const result = (0, classificationMetrics_1.computePrecisionRecallFScore)(example, options);
19
+ cache.set(example, result);
20
+ return result;
21
+ };
22
+ }
23
+ /**
24
+ * Creates matching precision, recall, and F-beta evaluators from a single set
25
+ * of options, so all three are computed with the same `average`, `beta`,
26
+ * `positiveLabel`, and `zeroDivision` settings. When the same `expected`/
27
+ * `output` example object is passed to all three evaluators, the underlying
28
+ * confusion matrix is only computed once and shared across them.
29
+ *
30
+ * @example
31
+ * ```typescript
32
+ * const { precision, recall, fScore } = createPrecisionRecallFScoreEvaluators({
33
+ * average: "weighted",
34
+ * });
35
+ * ```
36
+ */
37
+ function createPrecisionRecallFScoreEvaluators(options = {}) {
38
+ const { beta = 1 } = options;
39
+ const suffix = (0, classificationMetrics_1.getAverageMetricNameSuffix)(options);
40
+ const compute = createCachedComputer();
41
+ return {
42
+ precision: (0, createClassificationMetricEvaluator_1.createClassificationMetricEvaluator)(`precision${suffix}`, "precision", options, compute),
43
+ recall: (0, createClassificationMetricEvaluator_1.createClassificationMetricEvaluator)(`recall${suffix}`, "recall", options, compute),
44
+ fScore: (0, createClassificationMetricEvaluator_1.createClassificationMetricEvaluator)(`${(0, classificationMetrics_1.formatBetaForMetricName)(beta)}${suffix}`, "fScore", options, compute),
45
+ };
46
+ }
47
+ //# sourceMappingURL=createPrecisionRecallFScoreEvaluators.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createPrecisionRecallFScoreEvaluators.js","sourceRoot":"","sources":["../../../src/code/createPrecisionRecallFScoreEvaluators.ts"],"names":[],"mappings":";;AAsDA,sFA4BC;AAjFD,mEAIiC;AAMjC,+FAA4F;AAW5F;;;;GAIG;AACH,SAAS,oBAAoB;IAC3B,MAAM,KAAK,GAAG,IAAI,OAAO,EAAuC,CAAC;IACjE,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;QAC1B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,MAAM,GAAG,IAAA,oDAA4B,EAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC9D,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC3B,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAgB,qCAAqC,CAGnD,UAAwC,EAAE;IAE1C,MAAM,EAAE,IAAI,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAC7B,MAAM,MAAM,GAAG,IAAA,kDAA0B,EAAC,OAAO,CAAC,CAAC;IACnD,MAAM,OAAO,GAAG,oBAAoB,EAAE,CAAC;IACvC,OAAO;QACL,SAAS,EAAE,IAAA,yEAAmC,EAC5C,YAAY,MAAM,EAAE,EACpB,WAAW,EACX,OAAO,EACP,OAAO,CACR;QACD,MAAM,EAAE,IAAA,yEAAmC,EACzC,SAAS,MAAM,EAAE,EACjB,QAAQ,EACR,OAAO,EACP,OAAO,CACR;QACD,MAAM,EAAE,IAAA,yEAAmC,EACzC,GAAG,IAAA,+CAAuB,EAAC,IAAI,CAAC,GAAG,MAAM,EAAE,EAC3C,QAAQ,EACR,OAAO,EACP,OAAO,CACR;KACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,22 @@
1
+ import type { EvaluatorBase } from "../core/EvaluatorBase";
2
+ import type { ClassificationExample, PrecisionRecallFScoreOptions } from "./classificationMetrics";
3
+ /**
4
+ * Creates a code evaluator that computes recall: of the labels that actually
5
+ * belong to a given class, the fraction the model correctly predicted.
6
+ *
7
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
8
+ * `average` is at its default `"macro"` and labels are the numeric set
9
+ * `{0, 1}`) and multi-class classification (via the `average` strategy).
10
+ *
11
+ * @example
12
+ * ```typescript
13
+ * const recall = createRecallEvaluator();
14
+ * const result = await recall.evaluate({
15
+ * expected: ["cat", "dog", "cat", "bird"],
16
+ * output: ["cat", "cat", "cat", "bird"],
17
+ * });
18
+ * // { score: 2/3 }
19
+ * ```
20
+ */
21
+ export declare function createRecallEvaluator<RecordType extends ClassificationExample = ClassificationExample>(options?: PrecisionRecallFScoreOptions): EvaluatorBase<RecordType>;
22
+ //# sourceMappingURL=createRecallEvaluator.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createRecallEvaluator.d.ts","sourceRoot":"","sources":["../../../src/code/createRecallEvaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,KAAK,EACV,qBAAqB,EACrB,4BAA4B,EAC7B,MAAM,yBAAyB,CAAC;AAGjC;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,SAAS,qBAAqB,GAAG,qBAAqB,EAChE,OAAO,GAAE,4BAAiC,GAAG,aAAa,CAAC,UAAU,CAAC,CAOvE"}
@@ -0,0 +1,28 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createRecallEvaluator = createRecallEvaluator;
4
+ const classificationMetrics_1 = require("./classificationMetrics");
5
+ const createClassificationMetricEvaluator_1 = require("./createClassificationMetricEvaluator");
6
+ /**
7
+ * Creates a code evaluator that computes recall: of the labels that actually
8
+ * belong to a given class, the fraction the model correctly predicted.
9
+ *
10
+ * Supports binary classification (via `positiveLabel`, or auto-detected when
11
+ * `average` is at its default `"macro"` and labels are the numeric set
12
+ * `{0, 1}`) and multi-class classification (via the `average` strategy).
13
+ *
14
+ * @example
15
+ * ```typescript
16
+ * const recall = createRecallEvaluator();
17
+ * const result = await recall.evaluate({
18
+ * expected: ["cat", "dog", "cat", "bird"],
19
+ * output: ["cat", "cat", "cat", "bird"],
20
+ * });
21
+ * // { score: 2/3 }
22
+ * ```
23
+ */
24
+ function createRecallEvaluator(options = {}) {
25
+ const suffix = (0, classificationMetrics_1.getAverageMetricNameSuffix)(options);
26
+ return (0, createClassificationMetricEvaluator_1.createClassificationMetricEvaluator)(`recall${suffix}`, "recall", options);
27
+ }
28
+ //# sourceMappingURL=createRecallEvaluator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"createRecallEvaluator.js","sourceRoot":"","sources":["../../../src/code/createRecallEvaluator.ts"],"names":[],"mappings":";;AA0BA,sDASC;AAlCD,mEAAqE;AAKrE,+FAA4F;AAE5F;;;;;;;;;;;;;;;;;GAiBG;AACH,SAAgB,qBAAqB,CAEnC,UAAwC,EAAE;IAC1C,MAAM,MAAM,GAAG,IAAA,kDAA0B,EAAC,OAAO,CAAC,CAAC;IACnD,OAAO,IAAA,yEAAmC,EACxC,SAAS,MAAM,EAAE,EACjB,QAAQ,EACR,OAAO,CACR,CAAC;AACJ,CAAC"}
@@ -0,0 +1,7 @@
1
+ export * from "./classificationMetrics";
2
+ export * from "./createF1Evaluator";
3
+ export * from "./createFBetaEvaluator";
4
+ export * from "./createPrecisionEvaluator";
5
+ export * from "./createPrecisionRecallFScoreEvaluators";
6
+ export * from "./createRecallEvaluator";
7
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/code/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,yCAAyC,CAAC;AACxD,cAAc,yBAAyB,CAAC"}
@@ -0,0 +1,23 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./classificationMetrics"), exports);
18
+ __exportStar(require("./createF1Evaluator"), exports);
19
+ __exportStar(require("./createFBetaEvaluator"), exports);
20
+ __exportStar(require("./createPrecisionEvaluator"), exports);
21
+ __exportStar(require("./createPrecisionRecallFScoreEvaluators"), exports);
22
+ __exportStar(require("./createRecallEvaluator"), exports);
23
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/code/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0DAAwC;AACxC,sDAAoC;AACpC,yDAAuC;AACvC,6DAA2C;AAC3C,0EAAwD;AACxD,0DAAwC"}
@@ -1,3 +1,4 @@
1
+ export * from "./code";
1
2
  export * from "./llm";
2
3
  export * from "./template";
3
4
  export * from "./types";
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC;AACxB,cAAc,SAAS,CAAC;AACxB,cAAc,WAAW,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,QAAQ,CAAC;AACvB,cAAc,OAAO,CAAC;AACtB,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC;AACxB,cAAc,SAAS,CAAC;AACxB,cAAc,WAAW,CAAC"}
package/dist/src/index.js CHANGED
@@ -14,6 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
14
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./code"), exports);
17
18
  __exportStar(require("./llm"), exports);
18
19
  __exportStar(require("./template"), exports);
19
20
  __exportStar(require("./types"), exports);
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,wCAAsB;AACtB,6CAA2B;AAC3B,0CAAwB;AACxB,0CAAwB;AACxB,4CAA0B"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,yCAAuB;AACvB,wCAAsB;AACtB,6CAA2B;AAC3B,0CAAwB;AACxB,0CAAwB;AACxB,4CAA0B"}