@arizeai/phoenix-evals 2.7.0 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/README.md +40 -9
  3. package/dist/esm/decision/decideClassification.d.ts.map +1 -1
  4. package/dist/esm/decision/decideClassification.js +4 -4
  5. package/dist/esm/decision/decideClassification.js.map +1 -1
  6. package/dist/esm/helpers/toEvaluationResult.d.ts +3 -3
  7. package/dist/esm/helpers/toEvaluationResult.d.ts.map +1 -1
  8. package/dist/esm/helpers/toEvaluationResult.js +22 -3
  9. package/dist/esm/helpers/toEvaluationResult.js.map +1 -1
  10. package/dist/esm/llm/createCompletenessEvaluator.d.ts +1 -1
  11. package/dist/esm/llm/createCompletenessEvaluator.js +1 -1
  12. package/dist/esm/llm/createConcisenessEvaluator.d.ts +1 -1
  13. package/dist/esm/llm/createConcisenessEvaluator.js +1 -1
  14. package/dist/esm/llm/createCorrectnessEvaluator.d.ts +1 -1
  15. package/dist/esm/llm/createCorrectnessEvaluator.js +1 -1
  16. package/dist/esm/llm/createHallucinationEvaluator.d.ts +1 -1
  17. package/dist/esm/llm/createHallucinationEvaluator.js +1 -1
  18. package/dist/esm/llm/createPiiDetectionEvaluator.d.ts +1 -1
  19. package/dist/esm/llm/createPiiDetectionEvaluator.js +1 -1
  20. package/dist/esm/llm/createRefusalEvaluator.d.ts +1 -1
  21. package/dist/esm/llm/createRefusalEvaluator.js +1 -1
  22. package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts +1 -1
  23. package/dist/esm/llm/createRetrievalRelevanceEvaluator.js +1 -1
  24. package/dist/esm/llm/createToolInvocationEvaluator.d.ts +1 -1
  25. package/dist/esm/llm/createToolInvocationEvaluator.js +1 -1
  26. package/dist/esm/llm/createToolResponseHandlingEvaluator.d.ts +1 -1
  27. package/dist/esm/llm/createToolResponseHandlingEvaluator.js +1 -1
  28. package/dist/esm/llm/createToolSelectionEvaluator.d.ts +1 -1
  29. package/dist/esm/llm/createToolSelectionEvaluator.js +1 -1
  30. package/dist/esm/llm/createToxicityEvaluator.d.ts +1 -1
  31. package/dist/esm/llm/createToxicityEvaluator.js +1 -1
  32. package/dist/esm/llm/createUserFrictionEvaluator.d.ts +1 -1
  33. package/dist/esm/llm/createUserFrictionEvaluator.js +1 -1
  34. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  35. package/dist/src/decision/decideClassification.d.ts.map +1 -1
  36. package/dist/src/decision/decideClassification.js +4 -4
  37. package/dist/src/decision/decideClassification.js.map +1 -1
  38. package/dist/src/helpers/toEvaluationResult.d.ts +3 -3
  39. package/dist/src/helpers/toEvaluationResult.d.ts.map +1 -1
  40. package/dist/src/helpers/toEvaluationResult.js +22 -3
  41. package/dist/src/helpers/toEvaluationResult.js.map +1 -1
  42. package/dist/src/llm/createCompletenessEvaluator.d.ts +1 -1
  43. package/dist/src/llm/createCompletenessEvaluator.js +1 -1
  44. package/dist/src/llm/createConcisenessEvaluator.d.ts +1 -1
  45. package/dist/src/llm/createConcisenessEvaluator.js +1 -1
  46. package/dist/src/llm/createCorrectnessEvaluator.d.ts +1 -1
  47. package/dist/src/llm/createCorrectnessEvaluator.js +1 -1
  48. package/dist/src/llm/createHallucinationEvaluator.d.ts +1 -1
  49. package/dist/src/llm/createHallucinationEvaluator.js +1 -1
  50. package/dist/src/llm/createPiiDetectionEvaluator.d.ts +1 -1
  51. package/dist/src/llm/createPiiDetectionEvaluator.js +1 -1
  52. package/dist/src/llm/createRefusalEvaluator.d.ts +1 -1
  53. package/dist/src/llm/createRefusalEvaluator.js +1 -1
  54. package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts +1 -1
  55. package/dist/src/llm/createRetrievalRelevanceEvaluator.js +1 -1
  56. package/dist/src/llm/createToolInvocationEvaluator.d.ts +1 -1
  57. package/dist/src/llm/createToolInvocationEvaluator.js +1 -1
  58. package/dist/src/llm/createToolResponseHandlingEvaluator.d.ts +1 -1
  59. package/dist/src/llm/createToolResponseHandlingEvaluator.js +1 -1
  60. package/dist/src/llm/createToolSelectionEvaluator.d.ts +1 -1
  61. package/dist/src/llm/createToolSelectionEvaluator.js +1 -1
  62. package/dist/src/llm/createToxicityEvaluator.d.ts +1 -1
  63. package/dist/src/llm/createToxicityEvaluator.js +1 -1
  64. package/dist/src/llm/createUserFrictionEvaluator.d.ts +1 -1
  65. package/dist/src/llm/createUserFrictionEvaluator.js +1 -1
  66. package/dist/tsconfig.tsbuildinfo +1 -1
  67. package/docs/classification.mdx +2 -2
  68. package/package.json +12 -12
  69. package/src/decision/decideClassification.ts +4 -5
  70. package/src/helpers/toEvaluationResult.ts +27 -3
  71. package/src/llm/createCompletenessEvaluator.ts +1 -1
  72. package/src/llm/createConcisenessEvaluator.ts +1 -1
  73. package/src/llm/createCorrectnessEvaluator.ts +1 -1
  74. package/src/llm/createHallucinationEvaluator.ts +1 -1
  75. package/src/llm/createPiiDetectionEvaluator.ts +1 -1
  76. package/src/llm/createRefusalEvaluator.ts +1 -1
  77. package/src/llm/createRetrievalRelevanceEvaluator.ts +1 -1
  78. package/src/llm/createToolInvocationEvaluator.ts +1 -1
  79. package/src/llm/createToolResponseHandlingEvaluator.ts +1 -1
  80. package/src/llm/createToolSelectionEvaluator.ts +1 -1
  81. package/src/llm/createToxicityEvaluator.ts +1 -1
  82. package/src/llm/createUserFrictionEvaluator.ts +1 -1
package/CHANGELOG.md CHANGED
@@ -1,5 +1,17 @@
1
1
  # @arizeai/phoenix-evals
2
2
 
3
+ ## 2.8.0
4
+
5
+ ### Minor Changes
6
+
7
+ - d03cfdc: Support the Gemini Decisions API (`gemini-decision-1`) in classification evaluators. Pass `google.decisionModel("gemini-decision-1")` as the `model` to `createClassificationEvaluator` or any built-in. Requires `@ai-sdk/google` >= 4.0.118.
8
+
9
+ ## 2.7.1
10
+
11
+ ### Patch Changes
12
+
13
+ - 0128562: fix(phoenix-evals): keep `metadata` returned by custom evaluator functions wrapped with `createEvaluator`
14
+
3
15
  ## 2.7.0
4
16
 
5
17
  ### Minor Changes
package/README.md CHANGED
@@ -39,11 +39,13 @@ This package is built on [AI SDK](https://ai-sdk.dev/) v7, which is ESM-only. No
39
39
 
40
40
  The library provides a `createClassifier` function that allows you to create custom evaluators for different tasks like hallucination detection, relevance scoring, or any binary/multi-class classification.
41
41
 
42
+ A classifier can be backed by either a **language model** or a **decision model**. A language model returns a label with a written explanation, while a decision model answers through a typed choices API, returning a label with per-label probabilities (and a confidence when the provider reports one) but no explanation.
43
+
42
44
  ```typescript
43
- import { createClassifier } from "@arizeai/phoenix-evals/llm";
45
+ import { createClassificationEvaluator } from "@arizeai/phoenix-evals/llm";
44
46
  import { openai } from "@ai-sdk/openai";
45
47
 
46
- const model = openai("gpt-4o-mini");
48
+ const languageModel = openai("gpt-6-luna");
47
49
 
48
50
  const promptTemplate = `
49
51
  In this task, you will be presented with a query, a reference text and an answer. The answer is
@@ -69,14 +71,15 @@ Is the answer above factual or hallucinated based on the query and reference tex
69
71
  `;
70
72
 
71
73
  // Create the classifier
72
- const evaluator = await createClassifier({
73
- model,
74
+ const evaluator = await createClassificationEvaluator({
75
+ name: "hallucination",
76
+ model: languageModel,
74
77
  choices: { factual: 1, hallucinated: 0 },
75
78
  promptTemplate: promptTemplate,
76
79
  });
77
80
 
78
81
  // Use the classifier
79
- const result = await evaluator({
82
+ const result = await evaluator.evaluate({
80
83
  output: "Arize is not open source.",
81
84
  input: "Is Arize Phoenix Open Source?",
82
85
  reference:
@@ -84,7 +87,35 @@ const result = await evaluator({
84
87
  });
85
88
 
86
89
  console.log(result);
87
- // Output: { label: "hallucinated", score: 0 }
90
+ // Output: { label: "hallucinated", score: 0, explanation: "Arize Phoenix is open source, so the answer is inaccurate." }
91
+
92
+ // The same classifier can be backed by a decision model instead
93
+ const decisionModel = openai.decisionModel("gpt-6-luna");
94
+
95
+ const decisionEvaluator = await createClassificationEvaluator({
96
+ name: "hallucination",
97
+ model: decisionModel,
98
+ choices: { factual: 1, hallucinated: 0 },
99
+ promptTemplate: promptTemplate,
100
+ });
101
+
102
+ const decisionResult = await decisionEvaluator.evaluate({
103
+ output: "Arize is not open source.",
104
+ input: "Is Arize Phoenix Open Source?",
105
+ reference:
106
+ "Arize Phoenix is a platform for building and deploying AI applications. It is open source.",
107
+ });
108
+
109
+ console.log(decisionResult);
110
+ // Output: {
111
+ // label: "hallucinated",
112
+ // score: 0,
113
+ // metadata: {
114
+ // probabilities: { factual: 0.02, hallucinated: 0.98 },
115
+ // confidence: 0.96,
116
+ // modelId: "gpt-6-luna",
117
+ // },
118
+ // }
88
119
  ```
89
120
 
90
121
  See the complete example in [`examples/classifier_example.ts`](examples/classifier_example.ts).
@@ -118,7 +149,7 @@ import {
118
149
  } from "@arizeai/phoenix-evals/llm";
119
150
  import { openai } from "@ai-sdk/openai";
120
151
 
121
- const model = openai("gpt-4o-mini");
152
+ const model = openai("gpt-6-luna");
122
153
 
123
154
  // Faithfulness: checks if the output is grounded in the context
124
155
  const faithfulnessEvaluator = createFaithfulnessEvaluator({ model });
@@ -196,7 +227,7 @@ import { bindEvaluator } from "@arizeai/phoenix-evals";
196
227
  import { createFaithfulnessEvaluator } from "@arizeai/phoenix-evals/llm";
197
228
  import { openai } from "@ai-sdk/openai";
198
229
 
199
- const model = openai("gpt-4o-mini");
230
+ const model = openai("gpt-6-luna");
200
231
 
201
232
  type ExampleType = {
202
233
  question: string;
@@ -250,7 +281,7 @@ import {
250
281
 
251
282
  // Create your evaluator
252
283
  const faithfulnessEvaluator = createFaithfulnessEvaluator({
253
- model: openai("gpt-4o-mini"),
284
+ model: openai("gpt-6-luna"),
254
285
  });
255
286
 
256
287
  // Create a dataset for your experiment
@@ -1 +1 @@
1
- {"version":3,"file":"decideClassification.d.ts","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAe9D;;;;;;;;GAQG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA2BhC"}
1
+ {"version":3,"file":"decideClassification.d.ts","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAc9D;;;;;;;;GAQG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA2BhC"}
@@ -11,10 +11,10 @@ import { promptToDecisionState } from "./decisionPrompt.js";
11
11
  */
12
12
  const QUESTION_ID = "label";
13
13
  /**
14
- * The question's instructions. The rendered prompt, rubric included, is sent
15
- * as the state.
14
+ * Instructions sent with the choice question. The OpenAI Decisions API rejects
15
+ * a choice question that has no instructions.
16
16
  */
17
- const INSTRUCTIONS = "Read the prompt in the state and answer it by selecting exactly one of the choices.";
17
+ const CHOICE_INSTRUCTIONS = "Select the single best label for the input.";
18
18
  /**
19
19
  * Classify with an AI SDK decision model by asking one choice question whose
20
20
  * options are the labels. Decision models don't generate text, so the
@@ -32,7 +32,7 @@ export async function decideClassification(args) {
32
32
  questions: {
33
33
  [QUESTION_ID]: {
34
34
  type: "choice",
35
- instructions: INSTRUCTIONS,
35
+ instructions: CHOICE_INSTRUCTIONS,
36
36
  criteria: Object.fromEntries(labels.map((label) => [label, null])),
37
37
  },
38
38
  },
@@ -1 +1 @@
1
- {"version":3,"file":"decideClassification.js","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,IAAI,CAAC;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAKnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAC;AAE5B;;;GAGG;AACH,MAAM,YAAY,GAChB,qFAAqF,CAAC;AAExF;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAK1C;IACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC;QACvC,KAAK;QACL,KAAK,EAAE,qBAAqB,CAAC,MAAM,CAAC;QACpC,SAAS,EAAE;YACT,CAAC,WAAW,CAAC,EAAE;gBACb,IAAI,EAAE,QAAQ;gBACd,YAAY,EAAE,YAAY;gBAC1B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;aACnE;SACF;QACD,SAAS,EAAE,mBAAmB,CAAC;YAC7B,UAAU,EAAE,sBAAsB;YAClC,SAAS;SACV,CAAC;KACH,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC1D,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,EAAE;YACR,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnD,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO;SACjC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,gBAAyB;IAC9C,IAAI,OAAO,gBAAgB,KAAK,QAAQ,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;QACtE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACvD,MAAM,UAAU,GAAI,QAAqD;YACvE,EAAE,UAAU,EAAE,CAAC,WAAW,CAAC,CAAC;QAC9B,IAAI,OAAO,UAAU,KAAK,QAAQ;YAAE,OAAO,UAAU,CAAC;IACxD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
1
+ {"version":3,"file":"decideClassification.js","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,IAAI,CAAC;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAKnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAC;AAE5B;;;GAGG;AACH,MAAM,mBAAmB,GAAG,6CAA6C,CAAC;AAE1E;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAK1C;IACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC;QACvC,KAAK;QACL,KAAK,EAAE,qBAAqB,CAAC,MAAM,CAAC;QACpC,SAAS,EAAE;YACT,CAAC,WAAW,CAAC,EAAE;gBACb,IAAI,EAAE,QAAQ;gBACd,YAAY,EAAE,mBAAmB;gBACjC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;aACnE;SACF;QACD,SAAS,EAAE,mBAAmB,CAAC;YAC7B,UAAU,EAAE,sBAAsB;YAClC,SAAS;SACV,CAAC;KACH,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC1D,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,EAAE;YACR,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnD,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO;SACjC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,gBAAyB;IAC9C,IAAI,OAAO,gBAAgB,KAAK,QAAQ,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;QACtE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACvD,MAAM,UAAU,GAAI,QAAqD;YACvE,EAAE,UAAU,EAAE,CAAC,WAAW,CAAC,CAAC;QAC9B,IAAI,OAAO,UAAU,KAAK,QAAQ;YAAE,OAAO,UAAU,CAAC;IACxD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
@@ -8,7 +8,7 @@ import type { EvaluationResult } from "../types/index.js";
8
8
  *
9
9
  * - **Numbers**: Converted to `{ score: number }`
10
10
  * - **Strings**: Converted to `{ label: string }`
11
- * - **Objects**: Extracts `score`, `label`, and `explanation` properties if present
11
+ * - **Objects**: Extracts `score`, `label`, `explanation`, and `metadata` properties if present
12
12
  * - **Other types**: Returns an empty `EvaluationResult` object
13
13
  *
14
14
  * This is particularly useful when creating evaluators from functions that may
@@ -17,7 +17,7 @@ import type { EvaluationResult } from "../types/index.js";
17
17
  * @param result - The value to convert to an EvaluationResult. Can be:
18
18
  * - A number (converted to score)
19
19
  * - A string (converted to label)
20
- * - An object with optional `score`, `label`, and/or `explanation` properties
20
+ * - An object with optional `score`, `label`, `explanation`, and/or `metadata` properties
21
21
  * - Any other value (returns empty object)
22
22
  *
23
23
  * @returns An {@link EvaluationResult} object with extracted properties
@@ -37,7 +37,7 @@ import type { EvaluationResult } from "../types/index.js";
37
37
  * ```
38
38
  *
39
39
  * @example
40
- * Convert an object with all properties:
40
+ * Convert an object with multiple properties:
41
41
  * ```typescript
42
42
  * const result = toEvaluationResult({
43
43
  * score: 0.9,
@@ -1 +1 @@
1
- {"version":3,"file":"toEvaluationResult.d.ts","sourceRoot":"","sources":["../../../src/helpers/toEvaluationResult.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAuDjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CAyBpE"}
1
+ {"version":3,"file":"toEvaluationResult.d.ts","sourceRoot":"","sources":["../../../src/helpers/toEvaluationResult.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AA4EjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CA4BpE"}
@@ -40,6 +40,22 @@ function resultHasExplanation(result) {
40
40
  "explanation" in result &&
41
41
  typeof result.explanation === "string");
42
42
  }
43
+ /**
44
+ * Type guard to check if a value has a metadata property.
45
+ *
46
+ * @param result - The value to check
47
+ * @returns True if the value is an object with a metadata record
48
+ *
49
+ * @internal
50
+ */
51
+ function resultHasMetadata(result) {
52
+ return (typeof result === "object" &&
53
+ result !== null &&
54
+ "metadata" in result &&
55
+ typeof result.metadata === "object" &&
56
+ result.metadata !== null &&
57
+ !Array.isArray(result.metadata));
58
+ }
43
59
  /**
44
60
  * Converts an unknown value to an {@link EvaluationResult}.
45
61
  *
@@ -49,7 +65,7 @@ function resultHasExplanation(result) {
49
65
  *
50
66
  * - **Numbers**: Converted to `{ score: number }`
51
67
  * - **Strings**: Converted to `{ label: string }`
52
- * - **Objects**: Extracts `score`, `label`, and `explanation` properties if present
68
+ * - **Objects**: Extracts `score`, `label`, `explanation`, and `metadata` properties if present
53
69
  * - **Other types**: Returns an empty `EvaluationResult` object
54
70
  *
55
71
  * This is particularly useful when creating evaluators from functions that may
@@ -58,7 +74,7 @@ function resultHasExplanation(result) {
58
74
  * @param result - The value to convert to an EvaluationResult. Can be:
59
75
  * - A number (converted to score)
60
76
  * - A string (converted to label)
61
- * - An object with optional `score`, `label`, and/or `explanation` properties
77
+ * - An object with optional `score`, `label`, `explanation`, and/or `metadata` properties
62
78
  * - Any other value (returns empty object)
63
79
  *
64
80
  * @returns An {@link EvaluationResult} object with extracted properties
@@ -78,7 +94,7 @@ function resultHasExplanation(result) {
78
94
  * ```
79
95
  *
80
96
  * @example
81
- * Convert an object with all properties:
97
+ * Convert an object with multiple properties:
82
98
  * ```typescript
83
99
  * const result = toEvaluationResult({
84
100
  * score: 0.9,
@@ -126,6 +142,9 @@ export function toEvaluationResult(result) {
126
142
  if (resultHasExplanation(result)) {
127
143
  evaluationResult.explanation = result.explanation;
128
144
  }
145
+ if (resultHasMetadata(result)) {
146
+ evaluationResult.metadata = result.metadata;
147
+ }
129
148
  return evaluationResult;
130
149
  }
131
150
  return {};
@@ -1 +1 @@
1
- {"version":3,"file":"toEvaluationResult.js","sourceRoot":"","sources":["../../../src/helpers/toEvaluationResult.ts"],"names":[],"mappings":"AAEA;;;;;;;GAOG;AACH,SAAS,cAAc,CAAC,MAAe;IACrC,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,OAAO,IAAI,MAAM;QACjB,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CACjC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,cAAc,CAAC,MAAe;IACrC,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,OAAO,IAAI,MAAM;QACjB,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CACjC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,oBAAoB,CAC3B,MAAe;IAEf,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,aAAa,IAAI,MAAM;QACvB,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,CACvC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAe;IAChD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO;YACL,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO;YACL,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,MAAM,gBAAgB,GAAqB,EAAE,CAAC;QAC9C,IAAI,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,gBAAgB,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACxC,CAAC;QACD,IAAI,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,gBAAgB,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACxC,CAAC;QACD,IAAI,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,gBAAgB,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACpD,CAAC;QACD,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}
1
+ {"version":3,"file":"toEvaluationResult.js","sourceRoot":"","sources":["../../../src/helpers/toEvaluationResult.ts"],"names":[],"mappings":"AAEA;;;;;;;GAOG;AACH,SAAS,cAAc,CAAC,MAAe;IACrC,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,OAAO,IAAI,MAAM;QACjB,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CACjC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,cAAc,CAAC,MAAe;IACrC,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,OAAO,IAAI,MAAM;QACjB,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CACjC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,oBAAoB,CAC3B,MAAe;IAEf,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,aAAa,IAAI,MAAM;QACvB,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,CACvC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,iBAAiB,CACxB,MAAe;IAEf,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,UAAU,IAAI,MAAM;QACpB,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ;QACnC,MAAM,CAAC,QAAQ,KAAK,IAAI;QACxB,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAChC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAe;IAChD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO;YACL,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO;YACL,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,MAAM,gBAAgB,GAAqB,EAAE,CAAC;QAC9C,IAAI,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,gBAAgB,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACxC,CAAC;QACD,IAAI,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,gBAAgB,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACxC,CAAC;QACD,IAAI,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,gBAAgB,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACpD,CAAC;QACD,IAAI,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,gBAAgB,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC9C,CAAC;QACD,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}
@@ -34,7 +34,7 @@ export type CompletenessEvaluationRecord = {
34
34
  *
35
35
  * @example
36
36
  * ```ts
37
- * const evaluator = createCompletenessEvaluator({ model: openai("gpt-4o-mini") });
37
+ * const evaluator = createCompletenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
38
38
  * const result = await evaluator.evaluate({
39
39
  * conversation:
40
40
  * "User: Reset my password and update the billing address.\nAssistant: Your password has been reset.",
@@ -17,7 +17,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
17
17
  *
18
18
  * @example
19
19
  * ```ts
20
- * const evaluator = createCompletenessEvaluator({ model: openai("gpt-4o-mini") });
20
+ * const evaluator = createCompletenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
21
21
  * const result = await evaluator.evaluate({
22
22
  * conversation:
23
23
  * "User: Reset my password and update the billing address.\nAssistant: Your password has been reset.",
@@ -31,7 +31,7 @@ export type ConcisenessEvaluationRecord = {
31
31
  *
32
32
  * @example
33
33
  * ```ts
34
- * const evaluator = createConcisenessEvaluator({ model: openai("gpt-4o-mini") });
34
+ * const evaluator = createConcisenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
35
35
  * const result = await evaluator.evaluate({
36
36
  * input: "What is the capital of France?",
37
37
  * output: "Paris.",
@@ -18,7 +18,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
18
18
  *
19
19
  * @example
20
20
  * ```ts
21
- * const evaluator = createConcisenessEvaluator({ model: openai("gpt-4o-mini") });
21
+ * const evaluator = createConcisenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
22
22
  * const result = await evaluator.evaluate({
23
23
  * input: "What is the capital of France?",
24
24
  * output: "Paris.",
@@ -30,7 +30,7 @@ export type CorrectnessEvaluationRecord = {
30
30
  *
31
31
  * @example
32
32
  * ```ts
33
- * const evaluator = createCorrectnessEvaluator({ model: openai("gpt-4o-mini") });
33
+ * const evaluator = createCorrectnessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
34
34
  * const result = await evaluator.evaluate({
35
35
  * input: "What is the capital of France?",
36
36
  * output: "Paris is the capital of France.",
@@ -17,7 +17,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
17
17
  *
18
18
  * @example
19
19
  * ```ts
20
- * const evaluator = createCorrectnessEvaluator({ model: openai("gpt-4o-mini") });
20
+ * const evaluator = createCorrectnessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
21
21
  * const result = await evaluator.evaluate({
22
22
  * input: "What is the capital of France?",
23
23
  * output: "Paris is the capital of France.",
@@ -40,7 +40,7 @@ export interface HallucinationEvaluationRecord {
40
40
  *
41
41
  * @example
42
42
  * ```ts
43
- * const evaluator = createHallucinationEvaluator({ model: openai("gpt-4o-mini") });
43
+ * const evaluator = createHallucinationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
44
44
  * const result = await evaluator.evaluate({
45
45
  * input:
46
46
  * "User: What's our refund window?\nTool (lookup_policy): Refunds: 30 days from delivery.\nAssistant: 30 days from delivery.\nUser: And for electronics?",
@@ -20,7 +20,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
20
20
  *
21
21
  * @example
22
22
  * ```ts
23
- * const evaluator = createHallucinationEvaluator({ model: openai("gpt-4o-mini") });
23
+ * const evaluator = createHallucinationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
24
24
  * const result = await evaluator.evaluate({
25
25
  * input:
26
26
  * "User: What's our refund window?\nTool (lookup_policy): Refunds: 30 days from delivery.\nAssistant: 30 days from delivery.\nUser: And for electronics?",
@@ -34,7 +34,7 @@ export type PiiDetectionEvaluationRecord = {
34
34
  *
35
35
  * @example
36
36
  * ```ts
37
- * const evaluator = createPiiDetectionEvaluator({ model: openai("gpt-4o-mini") });
37
+ * const evaluator = createPiiDetectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
38
38
  * const result = await evaluator.evaluate({
39
39
  * conversation:
40
40
  * "User: Reset my account.\nAssistant: What email is on the account?\nUser: jane.doe@acme.com",
@@ -18,7 +18,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
18
18
  *
19
19
  * @example
20
20
  * ```ts
21
- * const evaluator = createPiiDetectionEvaluator({ model: openai("gpt-4o-mini") });
21
+ * const evaluator = createPiiDetectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
22
22
  * const result = await evaluator.evaluate({
23
23
  * conversation:
24
24
  * "User: Reset my account.\nAssistant: What email is on the account?\nUser: jane.doe@acme.com",
@@ -32,7 +32,7 @@ export type RefusalEvaluationRecord = {
32
32
  *
33
33
  * @example
34
34
  * ```ts
35
- * const evaluator = createRefusalEvaluator({ model: openai("gpt-4o-mini") });
35
+ * const evaluator = createRefusalEvaluator({ model: openai.decisionModel("gpt-6-luna") });
36
36
  * const result = await evaluator.evaluate({
37
37
  * input: "What is the capital of France?",
38
38
  * output: "I'm sorry, I can only help with technical questions.",
@@ -19,7 +19,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
19
19
  *
20
20
  * @example
21
21
  * ```ts
22
- * const evaluator = createRefusalEvaluator({ model: openai("gpt-4o-mini") });
22
+ * const evaluator = createRefusalEvaluator({ model: openai.decisionModel("gpt-6-luna") });
23
23
  * const result = await evaluator.evaluate({
24
24
  * input: "What is the capital of France?",
25
25
  * output: "I'm sorry, I can only help with technical questions.",
@@ -42,7 +42,7 @@ export interface RetrievalRelevanceEvaluationRecord {
42
42
  *
43
43
  * @example
44
44
  * ```ts
45
- * const evaluator = createRetrievalRelevanceEvaluator({ model: openai("gpt-4o-mini") });
45
+ * const evaluator = createRetrievalRelevanceEvaluator({ model: openai.decisionModel("gpt-6-luna") });
46
46
  * const result = await evaluator.evaluate({
47
47
  * input: "What is the capital of France?",
48
48
  * context: "Paris is the capital and largest city of France.",
@@ -21,7 +21,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
21
21
  *
22
22
  * @example
23
23
  * ```ts
24
- * const evaluator = createRetrievalRelevanceEvaluator({ model: openai("gpt-4o-mini") });
24
+ * const evaluator = createRetrievalRelevanceEvaluator({ model: openai.decisionModel("gpt-6-luna") });
25
25
  * const result = await evaluator.evaluate({
26
26
  * input: "What is the capital of France?",
27
27
  * context: "Paris is the capital and largest city of France.",
@@ -47,7 +47,7 @@ export type ToolInvocationEvaluationRecord = {
47
47
  *
48
48
  * @example
49
49
  * ```ts
50
- * const evaluator = createToolInvocationEvaluator({ model: openai("gpt-4o-mini") });
50
+ * const evaluator = createToolInvocationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
51
51
  *
52
52
  * // Example with JSON schema format for available tools
53
53
  * const result = await evaluator.evaluate({
@@ -24,7 +24,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
24
24
  *
25
25
  * @example
26
26
  * ```ts
27
- * const evaluator = createToolInvocationEvaluator({ model: openai("gpt-4o-mini") });
27
+ * const evaluator = createToolInvocationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
28
28
  *
29
29
  * // Example with JSON schema format for available tools
30
30
  * const result = await evaluator.evaluate({
@@ -47,7 +47,7 @@ export type ToolResponseHandlingEvaluationRecord = {
47
47
  *
48
48
  * @example
49
49
  * ```ts
50
- * const evaluator = createToolResponseHandlingEvaluator({ model: openai("gpt-4o-mini") });
50
+ * const evaluator = createToolResponseHandlingEvaluator({ model: openai.decisionModel("gpt-6-luna") });
51
51
  *
52
52
  * // Example: Correct extraction from tool result
53
53
  * const result = await evaluator.evaluate({
@@ -19,7 +19,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
19
19
  *
20
20
  * @example
21
21
  * ```ts
22
- * const evaluator = createToolResponseHandlingEvaluator({ model: openai("gpt-4o-mini") });
22
+ * const evaluator = createToolResponseHandlingEvaluator({ model: openai.decisionModel("gpt-6-luna") });
23
23
  *
24
24
  * // Example: Correct extraction from tool result
25
25
  * const result = await evaluator.evaluate({
@@ -48,7 +48,7 @@ export type ToolSelectionEvaluationRecord = {
48
48
  *
49
49
  * @example
50
50
  * ```ts
51
- * const evaluator = createToolSelectionEvaluator({ model: openai("gpt-4o-mini") });
51
+ * const evaluator = createToolSelectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
52
52
  *
53
53
  * const result = await evaluator.evaluate({
54
54
  * input: "User: What is the weather in San Francisco?",
@@ -25,7 +25,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
25
25
  *
26
26
  * @example
27
27
  * ```ts
28
- * const evaluator = createToolSelectionEvaluator({ model: openai("gpt-4o-mini") });
28
+ * const evaluator = createToolSelectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
29
29
  *
30
30
  * const result = await evaluator.evaluate({
31
31
  * input: "User: What is the weather in San Francisco?",
@@ -29,7 +29,7 @@ export type ToxicityEvaluationRecord = {
29
29
  *
30
30
  * @example
31
31
  * ```ts
32
- * const evaluator = createToxicityEvaluator({ model: openai("gpt-4o-mini") });
32
+ * const evaluator = createToxicityEvaluator({ model: openai.decisionModel("gpt-6-luna") });
33
33
  * const result = await evaluator.evaluate({
34
34
  * text: "You are a worthless idiot and everyone despises you.",
35
35
  * });
@@ -17,7 +17,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
17
17
  *
18
18
  * @example
19
19
  * ```ts
20
- * const evaluator = createToxicityEvaluator({ model: openai("gpt-4o-mini") });
20
+ * const evaluator = createToxicityEvaluator({ model: openai.decisionModel("gpt-6-luna") });
21
21
  * const result = await evaluator.evaluate({
22
22
  * text: "You are a worthless idiot and everyone despises you.",
23
23
  * });
@@ -36,7 +36,7 @@ export interface UserFrictionEvaluationRecord {
36
36
  *
37
37
  * @example
38
38
  * ```ts
39
- * const evaluator = createUserFrictionEvaluator({ model: openai("gpt-4o-mini") });
39
+ * const evaluator = createUserFrictionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
40
40
  * const result = await evaluator.evaluate({
41
41
  * conversation: "User: Show recent orders.\nAssistant: Here are last month's orders.",
42
42
  * userMessage: "No, I asked for this week.",
@@ -18,7 +18,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
18
18
  *
19
19
  * @example
20
20
  * ```ts
21
- * const evaluator = createUserFrictionEvaluator({ model: openai("gpt-4o-mini") });
21
+ * const evaluator = createUserFrictionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
22
22
  * const result = await evaluator.evaluate({
23
23
  * conversation: "User: Show recent orders.\nAssistant: Here are last month's orders.",
24
24
  * userMessage: "No, I asked for this week.",