@arizeai/phoenix-evals 2.7.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +40 -9
- package/dist/esm/decision/decideClassification.d.ts.map +1 -1
- package/dist/esm/decision/decideClassification.js +4 -4
- package/dist/esm/decision/decideClassification.js.map +1 -1
- package/dist/esm/helpers/toEvaluationResult.d.ts +3 -3
- package/dist/esm/helpers/toEvaluationResult.d.ts.map +1 -1
- package/dist/esm/helpers/toEvaluationResult.js +22 -3
- package/dist/esm/helpers/toEvaluationResult.js.map +1 -1
- package/dist/esm/llm/createCompletenessEvaluator.d.ts +1 -1
- package/dist/esm/llm/createCompletenessEvaluator.js +1 -1
- package/dist/esm/llm/createConcisenessEvaluator.d.ts +1 -1
- package/dist/esm/llm/createConcisenessEvaluator.js +1 -1
- package/dist/esm/llm/createCorrectnessEvaluator.d.ts +1 -1
- package/dist/esm/llm/createCorrectnessEvaluator.js +1 -1
- package/dist/esm/llm/createHallucinationEvaluator.d.ts +1 -1
- package/dist/esm/llm/createHallucinationEvaluator.js +1 -1
- package/dist/esm/llm/createPiiDetectionEvaluator.d.ts +1 -1
- package/dist/esm/llm/createPiiDetectionEvaluator.js +1 -1
- package/dist/esm/llm/createRefusalEvaluator.d.ts +1 -1
- package/dist/esm/llm/createRefusalEvaluator.js +1 -1
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.d.ts +1 -1
- package/dist/esm/llm/createRetrievalRelevanceEvaluator.js +1 -1
- package/dist/esm/llm/createToolInvocationEvaluator.d.ts +1 -1
- package/dist/esm/llm/createToolInvocationEvaluator.js +1 -1
- package/dist/esm/llm/createToolResponseHandlingEvaluator.d.ts +1 -1
- package/dist/esm/llm/createToolResponseHandlingEvaluator.js +1 -1
- package/dist/esm/llm/createToolSelectionEvaluator.d.ts +1 -1
- package/dist/esm/llm/createToolSelectionEvaluator.js +1 -1
- package/dist/esm/llm/createToxicityEvaluator.d.ts +1 -1
- package/dist/esm/llm/createToxicityEvaluator.js +1 -1
- package/dist/esm/llm/createUserFrictionEvaluator.d.ts +1 -1
- package/dist/esm/llm/createUserFrictionEvaluator.js +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/src/decision/decideClassification.d.ts.map +1 -1
- package/dist/src/decision/decideClassification.js +4 -4
- package/dist/src/decision/decideClassification.js.map +1 -1
- package/dist/src/helpers/toEvaluationResult.d.ts +3 -3
- package/dist/src/helpers/toEvaluationResult.d.ts.map +1 -1
- package/dist/src/helpers/toEvaluationResult.js +22 -3
- package/dist/src/helpers/toEvaluationResult.js.map +1 -1
- package/dist/src/llm/createCompletenessEvaluator.d.ts +1 -1
- package/dist/src/llm/createCompletenessEvaluator.js +1 -1
- package/dist/src/llm/createConcisenessEvaluator.d.ts +1 -1
- package/dist/src/llm/createConcisenessEvaluator.js +1 -1
- package/dist/src/llm/createCorrectnessEvaluator.d.ts +1 -1
- package/dist/src/llm/createCorrectnessEvaluator.js +1 -1
- package/dist/src/llm/createHallucinationEvaluator.d.ts +1 -1
- package/dist/src/llm/createHallucinationEvaluator.js +1 -1
- package/dist/src/llm/createPiiDetectionEvaluator.d.ts +1 -1
- package/dist/src/llm/createPiiDetectionEvaluator.js +1 -1
- package/dist/src/llm/createRefusalEvaluator.d.ts +1 -1
- package/dist/src/llm/createRefusalEvaluator.js +1 -1
- package/dist/src/llm/createRetrievalRelevanceEvaluator.d.ts +1 -1
- package/dist/src/llm/createRetrievalRelevanceEvaluator.js +1 -1
- package/dist/src/llm/createToolInvocationEvaluator.d.ts +1 -1
- package/dist/src/llm/createToolInvocationEvaluator.js +1 -1
- package/dist/src/llm/createToolResponseHandlingEvaluator.d.ts +1 -1
- package/dist/src/llm/createToolResponseHandlingEvaluator.js +1 -1
- package/dist/src/llm/createToolSelectionEvaluator.d.ts +1 -1
- package/dist/src/llm/createToolSelectionEvaluator.js +1 -1
- package/dist/src/llm/createToxicityEvaluator.d.ts +1 -1
- package/dist/src/llm/createToxicityEvaluator.js +1 -1
- package/dist/src/llm/createUserFrictionEvaluator.d.ts +1 -1
- package/dist/src/llm/createUserFrictionEvaluator.js +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/docs/classification.mdx +2 -2
- package/package.json +12 -12
- package/src/decision/decideClassification.ts +4 -5
- package/src/helpers/toEvaluationResult.ts +27 -3
- package/src/llm/createCompletenessEvaluator.ts +1 -1
- package/src/llm/createConcisenessEvaluator.ts +1 -1
- package/src/llm/createCorrectnessEvaluator.ts +1 -1
- package/src/llm/createHallucinationEvaluator.ts +1 -1
- package/src/llm/createPiiDetectionEvaluator.ts +1 -1
- package/src/llm/createRefusalEvaluator.ts +1 -1
- package/src/llm/createRetrievalRelevanceEvaluator.ts +1 -1
- package/src/llm/createToolInvocationEvaluator.ts +1 -1
- package/src/llm/createToolResponseHandlingEvaluator.ts +1 -1
- package/src/llm/createToolSelectionEvaluator.ts +1 -1
- package/src/llm/createToxicityEvaluator.ts +1 -1
- package/src/llm/createUserFrictionEvaluator.ts +1 -1
package/docs/classification.mdx
CHANGED
|
@@ -27,7 +27,7 @@ const result = await classify({
|
|
|
27
27
|
|
|
28
28
|
## Decision Models
|
|
29
29
|
|
|
30
|
-
Any classification evaluator, including every built-in, can run on an AI SDK decision model such as OpenAI's `gpt-6-luna` ([Decisions API](https://developers.openai.com/api/docs/guides/decisions)) or TypeSafe's Jev. Decision models pick one of the labels directly and return a probability for each, instead of generating text. They are much faster and cheaper than a chat model judge. With OpenAI, use `@ai-sdk/openai` 4.0.89 or later.
|
|
30
|
+
Any classification evaluator, including every built-in, can run on an AI SDK decision model such as OpenAI's `gpt-6-luna` ([Decisions API](https://developers.openai.com/api/docs/guides/decisions)), Google's `gemini-decision-1` ([Decisions API](https://aistudio.google.com/docs/decisions-api)), or TypeSafe's Jev. Decision models pick one of the labels directly and return a probability for each, instead of generating text. They are much faster and cheaper than a chat model judge. With OpenAI, use `@ai-sdk/openai` 4.0.89 or later; with Google, use `@ai-sdk/google` 4.0.118 or later.
|
|
31
31
|
|
|
32
32
|
```ts
|
|
33
33
|
import { openai } from "@ai-sdk/openai";
|
|
@@ -55,7 +55,7 @@ const result = await correctness.evaluate({
|
|
|
55
55
|
How it differs from a language model:
|
|
56
56
|
|
|
57
57
|
- **No explanation.** Decision models don't generate text, so `explanation` is always undefined.
|
|
58
|
-
- **Probabilities in metadata.** `metadata.probabilities` maps each label to its probability. `metadata.confidence` is included when the provider reports one (OpenAI and TypeSafe do).
|
|
58
|
+
- **Probabilities in metadata.** `metadata.probabilities` maps each label to its probability. `metadata.confidence` is included when the provider reports one (OpenAI, Google, and TypeSafe do).
|
|
59
59
|
- **Prompt shape.** The template is rendered the same way as for a language model, then sent as one plain-text state. Message roles are dropped and only text content is sent.
|
|
60
60
|
- **Choices.** Each provider sets its own limits on how many choices a question can have. A classification outside them fails with the provider's error.
|
|
61
61
|
- **Refusals.** A decision model may refuse to classify, for example when asked to infer sensitive attributes rejected by the chosen model. On a refusal, the evaluator throws the AI SDK's `Experimental_DecisionRefusalError`. In `runExperiment`, it's recorded as that evaluation's error.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@arizeai/phoenix-evals",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.8.0",
|
|
4
4
|
"description": "A library for running evaluations for AI use cases",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"arize",
|
|
@@ -66,27 +66,27 @@
|
|
|
66
66
|
}
|
|
67
67
|
},
|
|
68
68
|
"dependencies": {
|
|
69
|
-
"@ai-sdk/otel": "1.0.
|
|
70
|
-
"@arizeai/openinference-core": "^2.8.
|
|
71
|
-
"@opentelemetry/api": "^1.
|
|
72
|
-
"ai": "7.0.
|
|
69
|
+
"@ai-sdk/otel": "1.0.152",
|
|
70
|
+
"@arizeai/openinference-core": "^2.8.3",
|
|
71
|
+
"@opentelemetry/api": "^1.10.0",
|
|
72
|
+
"ai": "7.0.152",
|
|
73
73
|
"jsonpath-plus": "^10.4.0",
|
|
74
74
|
"mustache": "^4.2.0",
|
|
75
75
|
"zod": "^4.6.7"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
|
-
"@ai-sdk/google": "^4.0.
|
|
79
|
-
"@ai-sdk/openai": "
|
|
80
|
-
"@ai-sdk/typesafe-ai": "^3.0.
|
|
81
|
-
"@arizeai/openinference-instrumentation-openai": "^4.4.
|
|
82
|
-
"@opentelemetry/sdk-trace
|
|
78
|
+
"@ai-sdk/google": "^4.0.118",
|
|
79
|
+
"@ai-sdk/openai": "4.0.108",
|
|
80
|
+
"@ai-sdk/typesafe-ai": "^3.0.38",
|
|
81
|
+
"@arizeai/openinference-instrumentation-openai": "^4.4.6",
|
|
82
|
+
"@opentelemetry/sdk-trace": "^3.1.1",
|
|
83
83
|
"@types/mustache": "^4.2.6",
|
|
84
|
-
"@types/node": "^26.6.
|
|
84
|
+
"@types/node": "^26.6.6",
|
|
85
85
|
"msw": "^2.15.0",
|
|
86
86
|
"nock": "^14.0.17",
|
|
87
87
|
"tsx": "^4.23.15",
|
|
88
88
|
"typedoc": "^0.28.20",
|
|
89
|
-
"vitest": "^5.0
|
|
89
|
+
"vitest": "^5.1.0"
|
|
90
90
|
},
|
|
91
91
|
"engines": {
|
|
92
92
|
"node": ">=22.12"
|
|
@@ -18,11 +18,10 @@ import { promptToDecisionState } from "./decisionPrompt";
|
|
|
18
18
|
const QUESTION_ID = "label";
|
|
19
19
|
|
|
20
20
|
/**
|
|
21
|
-
*
|
|
22
|
-
*
|
|
21
|
+
* Instructions sent with the choice question. The OpenAI Decisions API rejects
|
|
22
|
+
* a choice question that has no instructions.
|
|
23
23
|
*/
|
|
24
|
-
const
|
|
25
|
-
"Read the prompt in the state and answer it by selecting exactly one of the choices.";
|
|
24
|
+
const CHOICE_INSTRUCTIONS = "Select the single best label for the input.";
|
|
26
25
|
|
|
27
26
|
/**
|
|
28
27
|
* Classify with an AI SDK decision model by asking one choice question whose
|
|
@@ -46,7 +45,7 @@ export async function decideClassification(args: {
|
|
|
46
45
|
questions: {
|
|
47
46
|
[QUESTION_ID]: {
|
|
48
47
|
type: "choice",
|
|
49
|
-
instructions:
|
|
48
|
+
instructions: CHOICE_INSTRUCTIONS,
|
|
50
49
|
criteria: Object.fromEntries(labels.map((label) => [label, null])),
|
|
51
50
|
},
|
|
52
51
|
},
|
|
@@ -53,6 +53,27 @@ function resultHasExplanation(
|
|
|
53
53
|
);
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
+
/**
|
|
57
|
+
* Type guard to check if a value has a metadata property.
|
|
58
|
+
*
|
|
59
|
+
* @param result - The value to check
|
|
60
|
+
* @returns True if the value is an object with a metadata record
|
|
61
|
+
*
|
|
62
|
+
* @internal
|
|
63
|
+
*/
|
|
64
|
+
function resultHasMetadata(
|
|
65
|
+
result: unknown
|
|
66
|
+
): result is { metadata: Record<string, unknown> } {
|
|
67
|
+
return (
|
|
68
|
+
typeof result === "object" &&
|
|
69
|
+
result !== null &&
|
|
70
|
+
"metadata" in result &&
|
|
71
|
+
typeof result.metadata === "object" &&
|
|
72
|
+
result.metadata !== null &&
|
|
73
|
+
!Array.isArray(result.metadata)
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
|
|
56
77
|
/**
|
|
57
78
|
* Converts an unknown value to an {@link EvaluationResult}.
|
|
58
79
|
*
|
|
@@ -62,7 +83,7 @@ function resultHasExplanation(
|
|
|
62
83
|
*
|
|
63
84
|
* - **Numbers**: Converted to `{ score: number }`
|
|
64
85
|
* - **Strings**: Converted to `{ label: string }`
|
|
65
|
-
* - **Objects**: Extracts `score`, `label`, and `
|
|
86
|
+
* - **Objects**: Extracts `score`, `label`, `explanation`, and `metadata` properties if present
|
|
66
87
|
* - **Other types**: Returns an empty `EvaluationResult` object
|
|
67
88
|
*
|
|
68
89
|
* This is particularly useful when creating evaluators from functions that may
|
|
@@ -71,7 +92,7 @@ function resultHasExplanation(
|
|
|
71
92
|
* @param result - The value to convert to an EvaluationResult. Can be:
|
|
72
93
|
* - A number (converted to score)
|
|
73
94
|
* - A string (converted to label)
|
|
74
|
-
* - An object with optional `score`, `label`, and/or `
|
|
95
|
+
* - An object with optional `score`, `label`, `explanation`, and/or `metadata` properties
|
|
75
96
|
* - Any other value (returns empty object)
|
|
76
97
|
*
|
|
77
98
|
* @returns An {@link EvaluationResult} object with extracted properties
|
|
@@ -91,7 +112,7 @@ function resultHasExplanation(
|
|
|
91
112
|
* ```
|
|
92
113
|
*
|
|
93
114
|
* @example
|
|
94
|
-
* Convert an object with
|
|
115
|
+
* Convert an object with multiple properties:
|
|
95
116
|
* ```typescript
|
|
96
117
|
* const result = toEvaluationResult({
|
|
97
118
|
* score: 0.9,
|
|
@@ -139,6 +160,9 @@ export function toEvaluationResult(result: unknown): EvaluationResult {
|
|
|
139
160
|
if (resultHasExplanation(result)) {
|
|
140
161
|
evaluationResult.explanation = result.explanation;
|
|
141
162
|
}
|
|
163
|
+
if (resultHasMetadata(result)) {
|
|
164
|
+
evaluationResult.metadata = result.metadata;
|
|
165
|
+
}
|
|
142
166
|
return evaluationResult;
|
|
143
167
|
}
|
|
144
168
|
return {};
|
|
@@ -44,7 +44,7 @@ export type CompletenessEvaluationRecord = {
|
|
|
44
44
|
*
|
|
45
45
|
* @example
|
|
46
46
|
* ```ts
|
|
47
|
-
* const evaluator = createCompletenessEvaluator({ model: openai("gpt-
|
|
47
|
+
* const evaluator = createCompletenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
48
48
|
* const result = await evaluator.evaluate({
|
|
49
49
|
* conversation:
|
|
50
50
|
* "User: Reset my password and update the billing address.\nAssistant: Your password has been reset.",
|
|
@@ -41,7 +41,7 @@ export type ConcisenessEvaluationRecord = {
|
|
|
41
41
|
*
|
|
42
42
|
* @example
|
|
43
43
|
* ```ts
|
|
44
|
-
* const evaluator = createConcisenessEvaluator({ model: openai("gpt-
|
|
44
|
+
* const evaluator = createConcisenessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
45
45
|
* const result = await evaluator.evaluate({
|
|
46
46
|
* input: "What is the capital of France?",
|
|
47
47
|
* output: "Paris.",
|
|
@@ -40,7 +40,7 @@ export type CorrectnessEvaluationRecord = {
|
|
|
40
40
|
*
|
|
41
41
|
* @example
|
|
42
42
|
* ```ts
|
|
43
|
-
* const evaluator = createCorrectnessEvaluator({ model: openai("gpt-
|
|
43
|
+
* const evaluator = createCorrectnessEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
44
44
|
* const result = await evaluator.evaluate({
|
|
45
45
|
* input: "What is the capital of France?",
|
|
46
46
|
* output: "Paris is the capital of France.",
|
|
@@ -50,7 +50,7 @@ export interface HallucinationEvaluationRecord {
|
|
|
50
50
|
*
|
|
51
51
|
* @example
|
|
52
52
|
* ```ts
|
|
53
|
-
* const evaluator = createHallucinationEvaluator({ model: openai("gpt-
|
|
53
|
+
* const evaluator = createHallucinationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
54
54
|
* const result = await evaluator.evaluate({
|
|
55
55
|
* input:
|
|
56
56
|
* "User: What's our refund window?\nTool (lookup_policy): Refunds: 30 days from delivery.\nAssistant: 30 days from delivery.\nUser: And for electronics?",
|
|
@@ -44,7 +44,7 @@ export type PiiDetectionEvaluationRecord = {
|
|
|
44
44
|
*
|
|
45
45
|
* @example
|
|
46
46
|
* ```ts
|
|
47
|
-
* const evaluator = createPiiDetectionEvaluator({ model: openai("gpt-
|
|
47
|
+
* const evaluator = createPiiDetectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
48
48
|
* const result = await evaluator.evaluate({
|
|
49
49
|
* conversation:
|
|
50
50
|
* "User: Reset my account.\nAssistant: What email is on the account?\nUser: jane.doe@acme.com",
|
|
@@ -42,7 +42,7 @@ export type RefusalEvaluationRecord = {
|
|
|
42
42
|
*
|
|
43
43
|
* @example
|
|
44
44
|
* ```ts
|
|
45
|
-
* const evaluator = createRefusalEvaluator({ model: openai("gpt-
|
|
45
|
+
* const evaluator = createRefusalEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
46
46
|
* const result = await evaluator.evaluate({
|
|
47
47
|
* input: "What is the capital of France?",
|
|
48
48
|
* output: "I'm sorry, I can only help with technical questions.",
|
|
@@ -53,7 +53,7 @@ export interface RetrievalRelevanceEvaluationRecord {
|
|
|
53
53
|
*
|
|
54
54
|
* @example
|
|
55
55
|
* ```ts
|
|
56
|
-
* const evaluator = createRetrievalRelevanceEvaluator({ model: openai("gpt-
|
|
56
|
+
* const evaluator = createRetrievalRelevanceEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
57
57
|
* const result = await evaluator.evaluate({
|
|
58
58
|
* input: "What is the capital of France?",
|
|
59
59
|
* context: "Paris is the capital and largest city of France.",
|
|
@@ -57,7 +57,7 @@ export type ToolInvocationEvaluationRecord = {
|
|
|
57
57
|
*
|
|
58
58
|
* @example
|
|
59
59
|
* ```ts
|
|
60
|
-
* const evaluator = createToolInvocationEvaluator({ model: openai("gpt-
|
|
60
|
+
* const evaluator = createToolInvocationEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
61
61
|
*
|
|
62
62
|
* // Example with JSON schema format for available tools
|
|
63
63
|
* const result = await evaluator.evaluate({
|
|
@@ -58,7 +58,7 @@ export type ToolResponseHandlingEvaluationRecord = {
|
|
|
58
58
|
*
|
|
59
59
|
* @example
|
|
60
60
|
* ```ts
|
|
61
|
-
* const evaluator = createToolResponseHandlingEvaluator({ model: openai("gpt-
|
|
61
|
+
* const evaluator = createToolResponseHandlingEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
62
62
|
*
|
|
63
63
|
* // Example: Correct extraction from tool result
|
|
64
64
|
* const result = await evaluator.evaluate({
|
|
@@ -58,7 +58,7 @@ export type ToolSelectionEvaluationRecord = {
|
|
|
58
58
|
*
|
|
59
59
|
* @example
|
|
60
60
|
* ```ts
|
|
61
|
-
* const evaluator = createToolSelectionEvaluator({ model: openai("gpt-
|
|
61
|
+
* const evaluator = createToolSelectionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
62
62
|
*
|
|
63
63
|
* const result = await evaluator.evaluate({
|
|
64
64
|
* input: "User: What is the weather in San Francisco?",
|
|
@@ -39,7 +39,7 @@ export type ToxicityEvaluationRecord = {
|
|
|
39
39
|
*
|
|
40
40
|
* @example
|
|
41
41
|
* ```ts
|
|
42
|
-
* const evaluator = createToxicityEvaluator({ model: openai("gpt-
|
|
42
|
+
* const evaluator = createToxicityEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
43
43
|
* const result = await evaluator.evaluate({
|
|
44
44
|
* text: "You are a worthless idiot and everyone despises you.",
|
|
45
45
|
* });
|
|
@@ -46,7 +46,7 @@ export interface UserFrictionEvaluationRecord {
|
|
|
46
46
|
*
|
|
47
47
|
* @example
|
|
48
48
|
* ```ts
|
|
49
|
-
* const evaluator = createUserFrictionEvaluator({ model: openai("gpt-
|
|
49
|
+
* const evaluator = createUserFrictionEvaluator({ model: openai.decisionModel("gpt-6-luna") });
|
|
50
50
|
* const result = await evaluator.evaluate({
|
|
51
51
|
* conversation: "User: Show recent orders.\nAssistant: Here are last month's orders.",
|
|
52
52
|
* userMessage: "No, I asked for this week.",
|