@arizeai/phoenix-evals 2.7.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +40 -9
- package/dist/esm/decision/decideClassification.d.ts.map +1 -1
- package/dist/esm/decision/decideClassification.js +4 -4
- package/dist/esm/decision/decideClassification.js.map +1 -1
- package/dist/esm/helpers/toEvaluationResult.d.ts +3 -3
- package/dist/esm/helpers/toEvaluationResult.d.ts.map +1 -1
- package/dist/esm/helpers/toEvaluationResult.js +22 -3
- package/dist/esm/helpers/toEvaluationResult.js.map +1 -1
- package/dist/esm/llm/createCompletenessEvaluator.d.ts +1 -1
- package/dist/esm/llm/createCompletenessEvaluator.js +1 -1
- package/dist/esm/llm/createConcisenessEvaluator.d.ts +1 -1
- package/dist/esm/llm/createConcisenessEvaluator.js +1 -1
- package/dist/esm/llm/createCorrectnessEvaluator.d.ts +1 -1
- package/dist/esm/llm/createCorrectnessEvaluator.js +1 -1
- package/dist/esm/llm/createHallucinationEvaluator.d.ts +1 -1
- package/dist/esm/llm/createHallucinationEvaluator.js +1 -1
- package/dist/esm/llm/createPiiDetectionEvaluator.d.ts +1 -1
- package/dist/esm/llm/createPiiDetectionEvaluator.js +1 -1
- package/dist/esm/llm/createRefusalEvaluator.d.ts +1 -1
- package/dist/esm/llm/createRefusalEvaluator.js +1 -1
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts +1 -1
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.js +1 -1
- package/dist/esm/llm/createToolInvocationEvaluator.d.ts +1 -1
- package/dist/esm/llm/createToolInvocationEvaluator.js +1 -1
- package/dist/esm/llm/createToolResponseHandlingEvaluator.d.ts +1 -1
- package/dist/esm/llm/createToolResponseHandlingEvaluator.js +1 -1
- package/dist/esm/llm/createToolSelectionEvaluator.d.ts +1 -1
- package/dist/esm/llm/createToolSelectionEvaluator.js +1 -1
- package/dist/esm/llm/createToxicityEvaluator.d.ts +1 -1
- package/dist/esm/llm/createToxicityEvaluator.js +1 -1
- package/dist/esm/llm/createUserFrictionEvaluator.d.ts +1 -1
- package/dist/esm/llm/createUserFrictionEvaluator.js +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/src/decision/decideClassification.d.ts.map +1 -1
- package/dist/src/decision/decideClassification.js +4 -4
- package/dist/src/decision/decideClassification.js.map +1 -1
- package/dist/src/helpers/toEvaluationResult.d.ts +3 -3
- package/dist/src/helpers/toEvaluationResult.d.ts.map +1 -1
- package/dist/src/helpers/toEvaluationResult.js +22 -3
- package/dist/src/helpers/toEvaluationResult.js.map +1 -1
- package/dist/src/llm/createCompletenessEvaluator.d.ts +1 -1
- package/dist/src/llm/createCompletenessEvaluator.js +1 -1
- package/dist/src/llm/createConcisenessEvaluator.d.ts +1 -1
- package/dist/src/llm/createConcisenessEvaluator.js +1 -1
- package/dist/src/llm/createCorrectnessEvaluator.d.ts +1 -1
- package/dist/src/llm/createCorrectnessEvaluator.js +1 -1
- package/dist/src/llm/createHallucinationEvaluator.d.ts +1 -1
- package/dist/src/llm/createHallucinationEvaluator.js +1 -1
- package/dist/src/llm/createPiiDetectionEvaluator.d.ts +1 -1
- package/dist/src/llm/createPiiDetectionEvaluator.js +1 -1
- package/dist/src/llm/createRefusalEvaluator.d.ts +1 -1
- package/dist/src/llm/createRefusalEvaluator.js +1 -1
- package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts +1 -1
- package/dist/src/llm/createRetrievalRelevanceEvaluator.js +1 -1
- package/dist/src/llm/createToolInvocationEvaluator.d.ts +1 -1
- package/dist/src/llm/createToolInvocationEvaluator.js +1 -1
- package/dist/src/llm/createToolResponseHandlingEvaluator.d.ts +1 -1
- package/dist/src/llm/createToolResponseHandlingEvaluator.js +1 -1
- package/dist/src/llm/createToolSelectionEvaluator.d.ts +1 -1
- package/dist/src/llm/createToolSelectionEvaluator.js +1 -1
- package/dist/src/llm/createToxicityEvaluator.d.ts +1 -1
- package/dist/src/llm/createToxicityEvaluator.js +1 -1
- package/dist/src/llm/createUserFrictionEvaluator.d.ts +1 -1
- package/dist/src/llm/createUserFrictionEvaluator.js +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/docs/classification.mdx +2 -2
- package/package.json +12 -12
- package/src/decision/decideClassification.ts +4 -5
- package/src/helpers/toEvaluationResult.ts +27 -3
- package/src/llm/createCompletenessEvaluator.ts +1 -1
- package/src/llm/createConcisenessEvaluator.ts +1 -1
- package/src/llm/createCorrectnessEvaluator.ts +1 -1
- package/src/llm/createHallucinationEvaluator.ts +1 -1
- package/src/llm/createPiiDetectionEvaluator.ts +1 -1
- package/src/llm/createRefusalEvaluator.ts +1 -1
- package/src/llm/createRetrievalRelevanceEvaluator.ts +1 -1
- package/src/llm/createToolInvocationEvaluator.ts +1 -1
- package/src/llm/createToolResponseHandlingEvaluator.ts +1 -1
- package/src/llm/createToolSelectionEvaluator.ts +1 -1
- package/src/llm/createToxicityEvaluator.ts +1 -1
- package/src/llm/createUserFrictionEvaluator.ts +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# @arizeai/phoenix-evals
|
|
2
2
|
|
|
3
|
+
## 2.8.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- d03cfdc: Support the Gemini Decisions API (`gemini-decision-1`) in classification evaluators. Pass `google.decisionModel("gemini-decision-1")` as the `model` to `createClassificationEvaluator` or any built-in. Requires `@ai-sdk/google` >= 4.0.118.
|
|
8
|
+
|
|
9
|
+
## 2.7.1
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- 0128562: fix(phoenix-evals): keep `metadata` returned by custom evaluator functions wrapped with `createEvaluator`
|
|
14
|
+
|
|
3
15
|
## 2.7.0
|
|
4
16
|
|
|
5
17
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -39,11 +39,13 @@ This package is built on [AI SDK](https://ai-sdk.dev/) v7, which is ESM-only. No
|
|
|
39
39
|
|
|
40
40
|
The library provides a `createClassifier` function that allows you to create custom evaluators for different tasks like hallucination detection, relevance scoring, or any binary/multi-class classification.
|
|
41
41
|
|
|
42
|
+
A classifier can be backed by either a **language model** or a **decision model**. A language model returns a label with a written explanation, while a decision model answers through a typed choices API, returning a label with per-label probabilities (and a confidence when the provider reports one) but no explanation.
|
|
43
|
+
|
|
42
44
|
```typescript
|
|
43
|
-
import {
|
|
45
|
+
import { createClassificationEvaluator } from "@arizeai/phoenix-evals/llm";
|
|
44
46
|
import { openai } from "@ai-sdk/openai";
|
|
45
47
|
|
|
46
|
-
const
|
|
48
|
+
const languageModel = openai("gpt-6-luna");
|
|
47
49
|
|
|
48
50
|
const promptTemplate = `
|
|
49
51
|
In this task, you will be presented with a query, a reference text and an answer. The answer is
|
|
@@ -69,14 +71,15 @@ Is the answer above factual or hallucinated based on the query and reference tex
|
|
|
69
71
|
`;
|
|
70
72
|
|
|
71
73
|
// Create the classifier
|
|
72
|
-
const evaluator = await
|
|
73
|
-
|
|
74
|
+
const evaluator = await createClassificationEvaluator({
|
|
75
|
+
name: "hallucination",
|
|
76
|
+
model: languageModel,
|
|
74
77
|
choices: { factual: 1, hallucinated: 0 },
|
|
75
78
|
promptTemplate: promptTemplate,
|
|
76
79
|
});
|
|
77
80
|
|
|
78
81
|
// Use the classifier
|
|
79
|
-
const result = await evaluator({
|
|
82
|
+
const result = await evaluator.evaluate({
|
|
80
83
|
output: "Arize is not open source.",
|
|
81
84
|
input: "Is Arize Phoenix Open Source?",
|
|
82
85
|
reference:
|
|
@@ -84,7 +87,35 @@ const result = await evaluator({
|
|
|
84
87
|
});
|
|
85
88
|
|
|
86
89
|
console.log(result);
|
|
87
|
-
// Output: { label: "hallucinated", score: 0 }
|
|
90
|
+
// Output: { label: "hallucinated", score: 0, explanation: "Arize Phoenix is open source, so the answer is inaccurate." }
|
|
91
|
+
|
|
92
|
+
// The same classifier can be backed by a decision model instead
|
|
93
|
+
const decisionModel = openai.decisionModel("gpt-6-luna");
|
|
94
|
+
|
|
95
|
+
const decisionEvaluator = await createClassificationEvaluator({
|
|
96
|
+
name: "hallucination",
|
|
97
|
+
model: decisionModel,
|
|
98
|
+
choices: { factual: 1, hallucinated: 0 },
|
|
99
|
+
promptTemplate: promptTemplate,
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
const decisionResult = await decisionEvaluator.evaluate({
|
|
103
|
+
output: "Arize is not open source.",
|
|
104
|
+
input: "Is Arize Phoenix Open Source?",
|
|
105
|
+
reference:
|
|
106
|
+
"Arize Phoenix is a platform for building and deploying AI applications. It is open source.",
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
console.log(decisionResult);
|
|
110
|
+
// Output: {
|
|
111
|
+
// label: "hallucinated",
|
|
112
|
+
// score: 0,
|
|
113
|
+
// metadata: {
|
|
114
|
+
// probabilities: { factual: 0.02, hallucinated: 0.98 },
|
|
115
|
+
// confidence: 0.96,
|
|
116
|
+
// modelId: "gpt-6-luna",
|
|
117
|
+
// },
|
|
118
|
+
// }
|
|
88
119
|
```
|
|
89
120
|
|
|
90
121
|
See the complete example in [`examples/classifier_example.ts`](examples/classifier_example.ts).
|
|
@@ -118,7 +149,7 @@ import {
|
|
|
118
149
|
} from "@arizeai/phoenix-evals/llm";
|
|
119
150
|
import { openai } from "@ai-sdk/openai";
|
|
120
151
|
|
|
121
|
-
const model = openai("gpt-
|
|
152
|
+
const model = openai("gpt-6-luna");
|
|
122
153
|
|
|
123
154
|
// Faithfulness: checks if the output is grounded in the context
|
|
124
155
|
const faithfulnessEvaluator = createFaithfulnessEvaluator({ model });
|
|
@@ -196,7 +227,7 @@ import { bindEvaluator } from "@arizeai/phoenix-evals";
|
|
|
196
227
|
import { createFaithfulnessEvaluator } from "@arizeai/phoenix-evals/llm";
|
|
197
228
|
import { openai } from "@ai-sdk/openai";
|
|
198
229
|
|
|
199
|
-
const model = openai("gpt-
|
|
230
|
+
const model = openai("gpt-6-luna");
|
|
200
231
|
|
|
201
232
|
type ExampleType = {
|
|
202
233
|
question: string;
|
|
@@ -250,7 +281,7 @@ import {
|
|
|
250
281
|
|
|
251
282
|
// Create your evaluator
|
|
252
283
|
const faithfulnessEvaluator = createFaithfulnessEvaluator({
|
|
253
|
-
model: openai("gpt-
|
|
284
|
+
model: openai("gpt-6-luna"),
|
|
254
285
|
});
|
|
255
286
|
|
|
256
287
|
// Create a dataset for your experiment
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"decideClassification.d.ts","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"decideClassification.d.ts","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAc9D;;;;;;;;GAQG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA2BhC"}
|
|
@@ -11,10 +11,10 @@ import { promptToDecisionState } from "./decisionPrompt.js";
|
|
|
11
11
|
*/
|
|
12
12
|
const QUESTION_ID = "label";
|
|
13
13
|
/**
|
|
14
|
-
*
|
|
15
|
-
*
|
|
14
|
+
* Instructions sent with the choice question. The OpenAI Decisions API rejects
|
|
15
|
+
* a choice question that has no instructions.
|
|
16
16
|
*/
|
|
17
|
-
const
|
|
17
|
+
const CHOICE_INSTRUCTIONS = "Select the single best label for the input.";
|
|
18
18
|
/**
|
|
19
19
|
* Classify with an AI SDK decision model by asking one choice question whose
|
|
20
20
|
* options are the labels. Decision models don't generate text, so the
|
|
@@ -32,7 +32,7 @@ export async function decideClassification(args) {
|
|
|
32
32
|
questions: {
|
|
33
33
|
[QUESTION_ID]: {
|
|
34
34
|
type: "choice",
|
|
35
|
-
instructions:
|
|
35
|
+
instructions: CHOICE_INSTRUCTIONS,
|
|
36
36
|
criteria: Object.fromEntries(labels.map((label) => [label, null])),
|
|
37
37
|
},
|
|
38
38
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"decideClassification.js","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,IAAI,CAAC;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAKnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAC;AAE5B;;;GAGG;AACH,MAAM,
|
|
1
|
+
{"version":3,"file":"decideClassification.js","sourceRoot":"","sources":["../../../src/decision/decideClassification.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,IAAI,CAAC;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAKnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAC;AAE5B;;;GAGG;AACH,MAAM,mBAAmB,GAAG,6CAA6C,CAAC;AAE1E;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAK1C;IACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC;QACvC,KAAK;QACL,KAAK,EAAE,qBAAqB,CAAC,MAAM,CAAC;QACpC,SAAS,EAAE;YACT,CAAC,WAAW,CAAC,EAAE;gBACb,IAAI,EAAE,QAAQ;gBACd,YAAY,EAAE,mBAAmB;gBACjC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;aACnE;SACF;QACD,SAAS,EAAE,mBAAmB,CAAC;YAC7B,UAAU,EAAE,sBAAsB;YAClC,SAAS;SACV,CAAC;KACH,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC1D,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,EAAE;YACR,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnD,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO;SACjC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,gBAAyB;IAC9C,IAAI,OAAO,gBAAgB,KAAK,QAAQ,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;QACtE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACvD,MAAM,UAAU,GAAI,QAAqD;YACvE,EAAE,UAAU,EAAE,CAAC,WAAW,CAAC,CAAC;QAC9B,IAAI,OAAO,UAAU,KAAK,QAAQ;YAAE,OAAO,UAAU,CAAC;IACxD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
|
@@ -8,7 +8,7 @@ import type { EvaluationResult } from "../types/index.js";
|
|
|
8
8
|
*
|
|
9
9
|
* - **Numbers**: Converted to `{ score: number }`
|
|
10
10
|
* - **Strings**: Converted to `{ label: string }`
|
|
11
|
-
* - **Objects**: Extracts `score`, `label`, and `
|
|
11
|
+
* - **Objects**: Extracts `score`, `label`, `explanation`, and `metadata` properties if present
|
|
12
12
|
* - **Other types**: Returns an empty `EvaluationResult` object
|
|
13
13
|
*
|
|
14
14
|
* This is particularly useful when creating evaluators from functions that may
|
|
@@ -17,7 +17,7 @@ import type { EvaluationResult } from "../types/index.js";
|
|
|
17
17
|
* @param result - The value to convert to an EvaluationResult. Can be:
|
|
18
18
|
* - A number (converted to score)
|
|
19
19
|
* - A string (converted to label)
|
|
20
|
-
* - An object with optional `score`, `label`, and/or `
|
|
20
|
+
* - An object with optional `score`, `label`, `explanation`, and/or `metadata` properties
|
|
21
21
|
* - Any other value (returns empty object)
|
|
22
22
|
*
|
|
23
23
|
* @returns An {@link EvaluationResult} object with extracted properties
|
|
@@ -37,7 +37,7 @@ import type { EvaluationResult } from "../types/index.js";
|
|
|
37
37
|
* ```
|
|
38
38
|
*
|
|
39
39
|
* @example
|
|
40
|
-
* Convert an object with
|
|
40
|
+
* Convert an object with multiple properties:
|
|
41
41
|
* ```typescript
|
|
42
42
|
* const result = toEvaluationResult({
|
|
43
43
|
* score: 0.9,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"toEvaluationResult.d.ts","sourceRoot":"","sources":["../../../src/helpers/toEvaluationResult.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"toEvaluationResult.d.ts","sourceRoot":"","sources":["../../../src/helpers/toEvaluationResult.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AA4EjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CA4BpE"}
|
|
@@ -40,6 +40,22 @@ function resultHasExplanation(result) {
|
|
|
40
40
|
"explanation" in result &&
|
|
41
41
|
typeof result.explanation === "string");
|
|
42
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Type guard to check if a value has a metadata property.
|
|
45
|
+
*
|
|
46
|
+
* @param result - The value to check
|
|
47
|
+
* @returns True if the value is an object with a metadata record
|
|
48
|
+
*
|
|
49
|
+
* @internal
|
|
50
|
+
*/
|
|
51
|
+
function resultHasMetadata(result) {
|
|
52
|
+
return (typeof result === "object" &&
|
|
53
|
+
result !== null &&
|
|
54
|
+
"metadata" in result &&
|
|
55
|
+
typeof result.metadata === "object" &&
|
|
56
|
+
result.metadata !== null &&
|
|
57
|
+
!Array.isArray(result.metadata));
|
|
58
|
+
}
|
|
43
59
|
/**
|
|
44
60
|
* Converts an unknown value to an {@link EvaluationResult}.
|
|
45
61
|
*
|
|
@@ -49,7 +65,7 @@ function resultHasExplanation(result) {
|
|
|
49
65
|
*
|
|
50
66
|
* - **Numbers**: Converted to `{ score: number }`
|
|
51
67
|
* - **Strings**: Converted to `{ label: string }`
|
|
52
|
-
* - **Objects**: Extracts `score`, `label`, and `
|
|
68
|
+
* - **Objects**: Extracts `score`, `label`, `explanation`, and `metadata` properties if present
|
|
53
69
|
* - **Other types**: Returns an empty `EvaluationResult` object
|
|
54
70
|
*
|
|
55
71
|
* This is particularly useful when creating evaluators from functions that may
|
|
@@ -58,7 +74,7 @@ function resultHasExplanation(result) {
|
|
|
58
74
|
* @param result - The value to convert to an EvaluationResult. Can be:
|
|
59
75
|
* - A number (converted to score)
|
|
60
76
|
* - A string (converted to label)
|
|
61
|
-
* - An object with optional `score`, `label`, and/or `
|
|
77
|
+
* - An object with optional `score`, `label`, `explanation`, and/or `metadata` properties
|
|
62
78
|
* - Any other value (returns empty object)
|
|
63
79
|
*
|
|
64
80
|
* @returns An {@link EvaluationResult} object with extracted properties
|
|
@@ -78,7 +94,7 @@ function resultHasExplanation(result) {
|
|
|
78
94
|
* ```
|
|
79
95
|
*
|
|
80
96
|
* @example
|
|
81
|
-
* Convert an object with
|
|
97
|
+
* Convert an object with multiple properties:
|
|
82
98
|
* ```typescript
|
|
83
99
|
* const result = toEvaluationResult({
|
|
84
100
|
* score: 0.9,
|
|
@@ -126,6 +142,9 @@ export function toEvaluationResult(result) {
|
|
|
126
142
|
if (resultHasExplanation(result)) {
|
|
127
143
|
evaluationResult.explanation = result.explanation;
|
|
128
144
|
}
|
|
145
|
+
if (resultHasMetadata(result)) {
|
|
146
|
+
evaluationResult.metadata = result.metadata;
|
|
147
|
+
}
|
|
129
148
|
return evaluationResult;
|
|
130
149
|
}
|
|
131
150
|
return {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"toEvaluationResult.js","sourceRoot":"","sources":["../../../src/helpers/toEvaluationResult.ts"],"names":[],"mappings":"AAEA;;;;;;;GAOG;AACH,SAAS,cAAc,CAAC,MAAe;IACrC,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,OAAO,IAAI,MAAM;QACjB,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CACjC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,cAAc,CAAC,MAAe;IACrC,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,OAAO,IAAI,MAAM;QACjB,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CACjC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,oBAAoB,CAC3B,MAAe;IAEf,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,aAAa,IAAI,MAAM;QACvB,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,CACvC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAe;IAChD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO;YACL,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO;YACL,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,MAAM,gBAAgB,GAAqB,EAAE,CAAC;QAC9C,IAAI,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,gBAAgB,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACxC,CAAC;QACD,IAAI,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,gBAAgB,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACxC,CAAC;QACD,IAAI,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,gBAAgB,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACpD,CAAC;QACD,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}
|
|
1
|
+
{"version":3,"file":"toEvaluationResult.js","sourceRoot":"","sources":["../../../src/helpers/toEvaluationResult.ts"],"names":[],"mappings":"AAEA;;;;;;;GAOG;AACH,SAAS,cAAc,CAAC,MAAe;IACrC,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,OAAO,IAAI,MAAM;QACjB,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CACjC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,cAAc,CAAC,MAAe;IACrC,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,OAAO,IAAI,MAAM;QACjB,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CACjC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,oBAAoB,CAC3B,MAAe;IAEf,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,aAAa,IAAI,MAAM;QACvB,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,CACvC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,iBAAiB,CACxB,MAAe;IAEf,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,UAAU,IAAI,MAAM;QACpB,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ;QACnC,MAAM,CAAC,QAAQ,KAAK,IAAI;QACxB,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAChC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAe;IAChD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO;YACL,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO;YACL,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,MAAM,gBAAgB,GAAqB,EAAE,CAAC;QAC9C,IAAI,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,gBAAgB,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACxC,CAAC;QACD,IAAI,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,gBAAgB,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACxC,CAAC;QACD,IAAI,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,gBAAgB,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACpD,CAAC;QACD,IAAI,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,gBAAgB,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC9C,CAAC;QACD,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}
|
|
@@ -34,7 +34,7 @@ export type CompletenessEvaluationRecord = {
|
|
|
34
34
|
*
|
|
35
35
|
* @example
|
|
36
36
|
* ```ts
|
|
37
|
-
* const evaluator = createCompletenessEvaluator({ model: openai("gpt-
|
|
37
|
+
* const evaluator = createCompletenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
38
38
|
* const result = await evaluator.evaluate({
|
|
39
39
|
* conversation:
|
|
40
40
|
* "User: Reset my password and update the billing address.\nAssistant: Your password has been reset.",
|
|
@@ -17,7 +17,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
17
17
|
*
|
|
18
18
|
* @example
|
|
19
19
|
* ```ts
|
|
20
|
-
* const evaluator = createCompletenessEvaluator({ model: openai("gpt-
|
|
20
|
+
* const evaluator = createCompletenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
21
21
|
* const result = await evaluator.evaluate({
|
|
22
22
|
* conversation:
|
|
23
23
|
* "User: Reset my password and update the billing address.\nAssistant: Your password has been reset.",
|
|
@@ -31,7 +31,7 @@ export type ConcisenessEvaluationRecord = {
|
|
|
31
31
|
*
|
|
32
32
|
* @example
|
|
33
33
|
* ```ts
|
|
34
|
-
* const evaluator = createConcisenessEvaluator({ model: openai("gpt-
|
|
34
|
+
* const evaluator = createConcisenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
35
35
|
* const result = await evaluator.evaluate({
|
|
36
36
|
* input: "What is the capital of France?",
|
|
37
37
|
* output: "Paris.",
|
|
@@ -18,7 +18,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
18
18
|
*
|
|
19
19
|
* @example
|
|
20
20
|
* ```ts
|
|
21
|
-
* const evaluator = createConcisenessEvaluator({ model: openai("gpt-
|
|
21
|
+
* const evaluator = createConcisenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
22
22
|
* const result = await evaluator.evaluate({
|
|
23
23
|
* input: "What is the capital of France?",
|
|
24
24
|
* output: "Paris.",
|
|
@@ -30,7 +30,7 @@ export type CorrectnessEvaluationRecord = {
|
|
|
30
30
|
*
|
|
31
31
|
* @example
|
|
32
32
|
* ```ts
|
|
33
|
-
* const evaluator = createCorrectnessEvaluator({ model: openai("gpt-
|
|
33
|
+
* const evaluator = createCorrectnessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
34
34
|
* const result = await evaluator.evaluate({
|
|
35
35
|
* input: "What is the capital of France?",
|
|
36
36
|
* output: "Paris is the capital of France.",
|
|
@@ -17,7 +17,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
17
17
|
*
|
|
18
18
|
* @example
|
|
19
19
|
* ```ts
|
|
20
|
-
* const evaluator = createCorrectnessEvaluator({ model: openai("gpt-
|
|
20
|
+
* const evaluator = createCorrectnessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
21
21
|
* const result = await evaluator.evaluate({
|
|
22
22
|
* input: "What is the capital of France?",
|
|
23
23
|
* output: "Paris is the capital of France.",
|
|
@@ -40,7 +40,7 @@ export interface HallucinationEvaluationRecord {
|
|
|
40
40
|
*
|
|
41
41
|
* @example
|
|
42
42
|
* ```ts
|
|
43
|
-
* const evaluator = createHallucinationEvaluator({ model: openai("gpt-
|
|
43
|
+
* const evaluator = createHallucinationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
44
44
|
* const result = await evaluator.evaluate({
|
|
45
45
|
* input:
|
|
46
46
|
* "User: What's our refund window?\nTool (lookup_policy): Refunds: 30 days from delivery.\nAssistant: 30 days from delivery.\nUser: And for electronics?",
|
|
@@ -20,7 +20,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
20
20
|
*
|
|
21
21
|
* @example
|
|
22
22
|
* ```ts
|
|
23
|
-
* const evaluator = createHallucinationEvaluator({ model: openai("gpt-
|
|
23
|
+
* const evaluator = createHallucinationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
24
24
|
* const result = await evaluator.evaluate({
|
|
25
25
|
* input:
|
|
26
26
|
* "User: What's our refund window?\nTool (lookup_policy): Refunds: 30 days from delivery.\nAssistant: 30 days from delivery.\nUser: And for electronics?",
|
|
@@ -34,7 +34,7 @@ export type PiiDetectionEvaluationRecord = {
|
|
|
34
34
|
*
|
|
35
35
|
* @example
|
|
36
36
|
* ```ts
|
|
37
|
-
* const evaluator = createPiiDetectionEvaluator({ model: openai("gpt-
|
|
37
|
+
* const evaluator = createPiiDetectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
38
38
|
* const result = await evaluator.evaluate({
|
|
39
39
|
* conversation:
|
|
40
40
|
* "User: Reset my account.\nAssistant: What email is on the account?\nUser: jane.doe@acme.com",
|
|
@@ -18,7 +18,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
18
18
|
*
|
|
19
19
|
* @example
|
|
20
20
|
* ```ts
|
|
21
|
-
* const evaluator = createPiiDetectionEvaluator({ model: openai("gpt-
|
|
21
|
+
* const evaluator = createPiiDetectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
22
22
|
* const result = await evaluator.evaluate({
|
|
23
23
|
* conversation:
|
|
24
24
|
* "User: Reset my account.\nAssistant: What email is on the account?\nUser: jane.doe@acme.com",
|
|
@@ -32,7 +32,7 @@ export type RefusalEvaluationRecord = {
|
|
|
32
32
|
*
|
|
33
33
|
* @example
|
|
34
34
|
* ```ts
|
|
35
|
-
* const evaluator = createRefusalEvaluator({ model: openai("gpt-
|
|
35
|
+
* const evaluator = createRefusalEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
36
36
|
* const result = await evaluator.evaluate({
|
|
37
37
|
* input: "What is the capital of France?",
|
|
38
38
|
* output: "I'm sorry, I can only help with technical questions.",
|
|
@@ -19,7 +19,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
19
19
|
*
|
|
20
20
|
* @example
|
|
21
21
|
* ```ts
|
|
22
|
-
* const evaluator = createRefusalEvaluator({ model: openai("gpt-
|
|
22
|
+
* const evaluator = createRefusalEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
23
23
|
* const result = await evaluator.evaluate({
|
|
24
24
|
* input: "What is the capital of France?",
|
|
25
25
|
* output: "I'm sorry, I can only help with technical questions.",
|
|
@@ -42,7 +42,7 @@ export interface RetrievalRelevanceEvaluationRecord {
|
|
|
42
42
|
*
|
|
43
43
|
* @example
|
|
44
44
|
* ```ts
|
|
45
|
-
* const evaluator = createRetrievalRelevanceEvaluator({ model: openai("gpt-
|
|
45
|
+
* const evaluator = createRetrievalRelevanceEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
46
46
|
* const result = await evaluator.evaluate({
|
|
47
47
|
* input: "What is the capital of France?",
|
|
48
48
|
* context: "Paris is the capital and largest city of France.",
|
|
@@ -21,7 +21,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
21
21
|
*
|
|
22
22
|
* @example
|
|
23
23
|
* ```ts
|
|
24
|
-
* const evaluator = createRetrievalRelevanceEvaluator({ model: openai("gpt-
|
|
24
|
+
* const evaluator = createRetrievalRelevanceEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
25
25
|
* const result = await evaluator.evaluate({
|
|
26
26
|
* input: "What is the capital of France?",
|
|
27
27
|
* context: "Paris is the capital and largest city of France.",
|
|
@@ -47,7 +47,7 @@ export type ToolInvocationEvaluationRecord = {
|
|
|
47
47
|
*
|
|
48
48
|
* @example
|
|
49
49
|
* ```ts
|
|
50
|
-
* const evaluator = createToolInvocationEvaluator({ model: openai("gpt-
|
|
50
|
+
* const evaluator = createToolInvocationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
51
51
|
*
|
|
52
52
|
* // Example with JSON schema format for available tools
|
|
53
53
|
* const result = await evaluator.evaluate({
|
|
@@ -24,7 +24,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
24
24
|
*
|
|
25
25
|
* @example
|
|
26
26
|
* ```ts
|
|
27
|
-
* const evaluator = createToolInvocationEvaluator({ model: openai("gpt-
|
|
27
|
+
* const evaluator = createToolInvocationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
28
28
|
*
|
|
29
29
|
* // Example with JSON schema format for available tools
|
|
30
30
|
* const result = await evaluator.evaluate({
|
|
@@ -47,7 +47,7 @@ export type ToolResponseHandlingEvaluationRecord = {
|
|
|
47
47
|
*
|
|
48
48
|
* @example
|
|
49
49
|
* ```ts
|
|
50
|
-
* const evaluator = createToolResponseHandlingEvaluator({ model: openai("gpt-
|
|
50
|
+
* const evaluator = createToolResponseHandlingEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
51
51
|
*
|
|
52
52
|
* // Example: Correct extraction from tool result
|
|
53
53
|
* const result = await evaluator.evaluate({
|
|
@@ -19,7 +19,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
19
19
|
*
|
|
20
20
|
* @example
|
|
21
21
|
* ```ts
|
|
22
|
-
* const evaluator = createToolResponseHandlingEvaluator({ model: openai("gpt-
|
|
22
|
+
* const evaluator = createToolResponseHandlingEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
23
23
|
*
|
|
24
24
|
* // Example: Correct extraction from tool result
|
|
25
25
|
* const result = await evaluator.evaluate({
|
|
@@ -48,7 +48,7 @@ export type ToolSelectionEvaluationRecord = {
|
|
|
48
48
|
*
|
|
49
49
|
* @example
|
|
50
50
|
* ```ts
|
|
51
|
-
* const evaluator = createToolSelectionEvaluator({ model: openai("gpt-
|
|
51
|
+
* const evaluator = createToolSelectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
52
52
|
*
|
|
53
53
|
* const result = await evaluator.evaluate({
|
|
54
54
|
* input: "User: What is the weather in San Francisco?",
|
|
@@ -25,7 +25,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
25
25
|
*
|
|
26
26
|
* @example
|
|
27
27
|
* ```ts
|
|
28
|
-
* const evaluator = createToolSelectionEvaluator({ model: openai("gpt-
|
|
28
|
+
* const evaluator = createToolSelectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
29
29
|
*
|
|
30
30
|
* const result = await evaluator.evaluate({
|
|
31
31
|
* input: "User: What is the weather in San Francisco?",
|
|
@@ -29,7 +29,7 @@ export type ToxicityEvaluationRecord = {
|
|
|
29
29
|
*
|
|
30
30
|
* @example
|
|
31
31
|
* ```ts
|
|
32
|
-
* const evaluator = createToxicityEvaluator({ model: openai("gpt-
|
|
32
|
+
* const evaluator = createToxicityEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
33
33
|
* const result = await evaluator.evaluate({
|
|
34
34
|
* text: "You are a worthless idiot and everyone despises you.",
|
|
35
35
|
* });
|
|
@@ -17,7 +17,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
17
17
|
*
|
|
18
18
|
* @example
|
|
19
19
|
* ```ts
|
|
20
|
-
* const evaluator = createToxicityEvaluator({ model: openai("gpt-
|
|
20
|
+
* const evaluator = createToxicityEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
21
21
|
* const result = await evaluator.evaluate({
|
|
22
22
|
* text: "You are a worthless idiot and everyone despises you.",
|
|
23
23
|
* });
|
|
@@ -36,7 +36,7 @@ export interface UserFrictionEvaluationRecord {
|
|
|
36
36
|
*
|
|
37
37
|
* @example
|
|
38
38
|
* ```ts
|
|
39
|
-
* const evaluator = createUserFrictionEvaluator({ model: openai("gpt-
|
|
39
|
+
* const evaluator = createUserFrictionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
40
40
|
* const result = await evaluator.evaluate({
|
|
41
41
|
* conversation: "User: Show recent orders.\nAssistant: Here are last month's orders.",
|
|
42
42
|
* userMessage: "No, I asked for this week.",
|
|
@@ -18,7 +18,7 @@ import { createClassificationEvaluator } from "./createClassificationEvaluator.j
|
|
|
18
18
|
*
|
|
19
19
|
* @example
|
|
20
20
|
* ```ts
|
|
21
|
-
* const evaluator = createUserFrictionEvaluator({ model: openai("gpt-
|
|
21
|
+
* const evaluator = createUserFrictionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
22
22
|
* const result = await evaluator.evaluate({
|
|
23
23
|
* conversation: "User: Show recent orders.\nAssistant: Here are last month's orders.",
|
|
24
24
|
* userMessage: "No, I asked for this week.",
|