@arizeai/phoenix-evals 2.6.0 → 2.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/esm/decision/decideClassification.d.ts +20 -0
- package/dist/esm/decision/decideClassification.d.ts.map +1 -0
- package/dist/esm/decision/decideClassification.js +72 -0
- package/dist/esm/decision/decideClassification.js.map +1 -0
- package/dist/esm/decision/decisionPrompt.d.ts +8 -0
- package/dist/esm/decision/decisionPrompt.d.ts.map +1 -0
- package/dist/esm/decision/decisionPrompt.js +66 -0
- package/dist/esm/decision/decisionPrompt.js.map +1 -0
- package/dist/esm/helpers/toEvaluationResult.d.ts +3 -3
- package/dist/esm/helpers/toEvaluationResult.d.ts.map +1 -1
- package/dist/esm/helpers/toEvaluationResult.js +22 -3
- package/dist/esm/helpers/toEvaluationResult.js.map +1 -1
- package/dist/esm/llm/ClassificationEvaluator.d.ts +2 -2
- package/dist/esm/llm/ClassificationEvaluator.d.ts.map +1 -1
- package/dist/esm/llm/ClassificationEvaluator.js.map +1 -1
- package/dist/esm/llm/generateClassification.d.ts +5 -5
- package/dist/esm/llm/generateClassification.d.ts.map +1 -1
- package/dist/esm/llm/generateClassification.js +20 -71
- package/dist/esm/llm/generateClassification.js.map +1 -1
- package/dist/esm/telemetry/index.d.ts +14 -0
- package/dist/esm/telemetry/index.d.ts.map +1 -1
- package/dist/esm/telemetry/index.js +15 -0
- package/dist/esm/telemetry/index.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/esm/types/evals.d.ts +8 -6
- package/dist/esm/types/evals.d.ts.map +1 -1
- package/dist/esm/utils/index.d.ts +1 -0
- package/dist/esm/utils/index.d.ts.map +1 -1
- package/dist/esm/utils/index.js +1 -0
- package/dist/esm/utils/index.js.map +1 -1
- package/dist/esm/utils/isDecisionModel.d.ts +18 -0
- package/dist/esm/utils/isDecisionModel.d.ts.map +1 -0
- package/dist/esm/utils/isDecisionModel.js +13 -0
- package/dist/esm/utils/isDecisionModel.js.map +1 -0
- package/dist/src/decision/decideClassification.d.ts +20 -0
- package/dist/src/decision/decideClassification.d.ts.map +1 -0
- package/dist/src/decision/decideClassification.js +71 -0
- package/dist/src/decision/decideClassification.js.map +1 -0
- package/dist/src/decision/decisionPrompt.d.ts +8 -0
- package/dist/src/decision/decisionPrompt.d.ts.map +1 -0
- package/dist/src/decision/decisionPrompt.js +69 -0
- package/dist/src/decision/decisionPrompt.js.map +1 -0
- package/dist/src/helpers/toEvaluationResult.d.ts +3 -3
- package/dist/src/helpers/toEvaluationResult.d.ts.map +1 -1
- package/dist/src/helpers/toEvaluationResult.js +22 -3
- package/dist/src/helpers/toEvaluationResult.js.map +1 -1
- package/dist/src/llm/ClassificationEvaluator.d.ts +2 -2
- package/dist/src/llm/ClassificationEvaluator.d.ts.map +1 -1
- package/dist/src/llm/generateClassification.d.ts +5 -5
- package/dist/src/llm/generateClassification.d.ts.map +1 -1
- package/dist/src/llm/generateClassification.js +18 -70
- package/dist/src/llm/generateClassification.js.map +1 -1
- package/dist/src/telemetry/index.d.ts +14 -0
- package/dist/src/telemetry/index.d.ts.map +1 -1
- package/dist/src/telemetry/index.js +17 -0
- package/dist/src/telemetry/index.js.map +1 -1
- package/dist/src/types/evals.d.ts +8 -6
- package/dist/src/types/evals.d.ts.map +1 -1
- package/dist/src/utils/index.d.ts +1 -0
- package/dist/src/utils/index.d.ts.map +1 -1
- package/dist/src/utils/index.js +1 -0
- package/dist/src/utils/index.js.map +1 -1
- package/dist/src/utils/isDecisionModel.d.ts +18 -0
- package/dist/src/utils/isDecisionModel.d.ts.map +1 -0
- package/dist/src/utils/isDecisionModel.js +16 -0
- package/dist/src/utils/isDecisionModel.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/docs/classification.mdx +43 -1
- package/package.json +14 -14
- package/src/decision/decideClassification.ts +85 -0
- package/src/decision/decisionPrompt.ts +70 -0
- package/src/helpers/toEvaluationResult.ts +27 -3
- package/src/llm/ClassificationEvaluator.ts +2 -2
- package/src/llm/generateClassification.ts +20 -82
- package/src/telemetry/index.ts +21 -0
- package/src/types/evals.ts +8 -6
- package/src/utils/index.ts +1 -0
- package/src/utils/isDecisionModel.ts +29 -0
- package/dist/esm/utils/isEvaluationModel.d.ts +0 -13
- package/dist/esm/utils/isEvaluationModel.d.ts.map +0 -1
- package/dist/esm/utils/isEvaluationModel.js +0 -11
- package/dist/esm/utils/isEvaluationModel.js.map +0 -1
- package/dist/src/utils/isEvaluationModel.d.ts +0 -13
- package/dist/src/utils/isEvaluationModel.d.ts.map +0 -1
- package/dist/src/utils/isEvaluationModel.js +0 -14
- package/dist/src/utils/isEvaluationModel.js.map +0 -1
- package/src/utils/isEvaluationModel.ts +0 -23
package/docs/classification.mdx
CHANGED
|
@@ -25,9 +25,49 @@ const result = await classify({
|
|
|
25
25
|
});
|
|
26
26
|
```
|
|
27
27
|
|
|
28
|
+
## Decision Models
|
|
29
|
+
|
|
30
|
+
Any classification evaluator, including every built-in, can run on an AI SDK decision model such as OpenAI's `gpt-6-luna` ([Decisions API](https://developers.openai.com/api/docs/guides/decisions)) or TypeSafe's Jev. Decision models pick one of the labels directly and return a probability for each, instead of generating text. They are much faster and cheaper than a chat model judge. With OpenAI, use `@ai-sdk/openai` 4.0.89 or later.
|
|
31
|
+
|
|
32
|
+
```ts
|
|
33
|
+
import { openai } from "@ai-sdk/openai";
|
|
34
|
+
import { createCorrectnessEvaluator } from "@arizeai/phoenix-evals";
|
|
35
|
+
|
|
36
|
+
const correctness = createCorrectnessEvaluator({
|
|
37
|
+
model: openai.decisionModel("gpt-6-luna"),
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
const result = await correctness.evaluate({
|
|
41
|
+
input: "What is the capital of Australia?",
|
|
42
|
+
output: "Sydney.",
|
|
43
|
+
});
|
|
44
|
+
// {
|
|
45
|
+
// label: "incorrect",
|
|
46
|
+
// score: 0,
|
|
47
|
+
// metadata: {
|
|
48
|
+
// probabilities: { correct: 0.02, incorrect: 0.98 },
|
|
49
|
+
// confidence: 0.96,
|
|
50
|
+
// modelId: "gpt-6-luna",
|
|
51
|
+
// },
|
|
52
|
+
// }
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
How it differs from a language model:
|
|
56
|
+
|
|
57
|
+
- **No explanation.** Decision models don't generate text, so `explanation` is always undefined.
|
|
58
|
+
- **Probabilities in metadata.** `metadata.probabilities` maps each label to its probability. `metadata.confidence` is included when the provider reports one (OpenAI and TypeSafe do).
|
|
59
|
+
- **Prompt shape.** The template is rendered the same way as for a language model, then sent as one plain-text state. Message roles are dropped and only text content is sent.
|
|
60
|
+
- **Choices.** Each provider sets its own limits on how many choices a question can have. A classification outside them fails with the provider's error.
|
|
61
|
+
- **Refusals.** A decision model may refuse to classify, for example when asked to infer sensitive attributes rejected by the chosen model. On a refusal, the evaluator throws the AI SDK's `Experimental_DecisionRefusalError`. In `runExperiment`, it's recorded as that evaluation's error.
|
|
62
|
+
- **Plain model IDs.** A string model ID is always treated as a language model. Use the provider's `decisionModel()` to get a decision model.
|
|
63
|
+
|
|
64
|
+
<Warning>
|
|
65
|
+
Decision models can't work through a problem step by step. They do well on judgments you can read off the text, such as relevance, faithfulness, tool selection, and toxicity. They're unreliable for checking answers that need a calculation, such as unit conversions or arithmetic, and their probabilities can be confidently wrong on those. Use a language model for those evaluations.
|
|
66
|
+
</Warning>
|
|
67
|
+
|
|
28
68
|
## Lower-Level API
|
|
29
69
|
|
|
30
|
-
Use `generateClassification` directly when you already have a rendered prompt and only need structured label generation.
|
|
70
|
+
Use `generateClassification` directly when you already have a rendered prompt and only need structured label generation. It also accepts a decision model, in which case the rendered prompt is sent as one plain-text state.
|
|
31
71
|
|
|
32
72
|
<section className="hidden" data-agent-context="source-map" aria-label="Source map">
|
|
33
73
|
<h2>Source Map</h2>
|
|
@@ -35,6 +75,8 @@ Use `generateClassification` directly when you already have a rendered prompt an
|
|
|
35
75
|
<li><code>src/llm/createClassifierFn.ts</code></li>
|
|
36
76
|
<li><code>src/llm/createClassificationEvaluator.ts</code></li>
|
|
37
77
|
<li><code>src/llm/generateClassification.ts</code></li>
|
|
78
|
+
<li><code>src/decision/decideClassification.ts</code></li>
|
|
79
|
+
<li><code>src/decision/decisionPrompt.ts</code></li>
|
|
38
80
|
<li><code>src/types/evals.ts</code></li>
|
|
39
81
|
</ul>
|
|
40
82
|
</section>
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@arizeai/phoenix-evals",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.7.1",
|
|
4
4
|
"description": "A library for running evaluations for AI use cases",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"arize",
|
|
@@ -66,27 +66,27 @@
|
|
|
66
66
|
}
|
|
67
67
|
},
|
|
68
68
|
"dependencies": {
|
|
69
|
-
"@ai-sdk/otel": "
|
|
70
|
-
"@arizeai/openinference-core": "^2.
|
|
71
|
-
"@opentelemetry/api": "^1.
|
|
72
|
-
"ai": "
|
|
69
|
+
"@ai-sdk/otel": "1.0.152",
|
|
70
|
+
"@arizeai/openinference-core": "^2.8.1",
|
|
71
|
+
"@opentelemetry/api": "^1.10.0",
|
|
72
|
+
"ai": "7.0.152",
|
|
73
73
|
"jsonpath-plus": "^10.4.0",
|
|
74
74
|
"mustache": "^4.2.0",
|
|
75
|
-
"zod": "^4.6.
|
|
75
|
+
"zod": "^4.6.7"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
|
-
"@ai-sdk/google": "^4.0.
|
|
79
|
-
"@ai-sdk/openai": "^4.0.
|
|
80
|
-
"@ai-sdk/typesafe-ai": "^3.0.
|
|
81
|
-
"@arizeai/openinference-instrumentation-openai": "^4.2
|
|
82
|
-
"@opentelemetry/sdk-trace-node": "^2.
|
|
78
|
+
"@ai-sdk/google": "^4.0.104",
|
|
79
|
+
"@ai-sdk/openai": "^4.0.101",
|
|
80
|
+
"@ai-sdk/typesafe-ai": "^3.0.26",
|
|
81
|
+
"@arizeai/openinference-instrumentation-openai": "^4.4.2",
|
|
82
|
+
"@opentelemetry/sdk-trace-node": "^2.12.0",
|
|
83
83
|
"@types/mustache": "^4.2.6",
|
|
84
|
-
"@types/node": "^26.6.
|
|
84
|
+
"@types/node": "^26.6.5",
|
|
85
85
|
"msw": "^2.15.0",
|
|
86
86
|
"nock": "^14.0.17",
|
|
87
|
-
"tsx": "^4.23.
|
|
87
|
+
"tsx": "^4.23.15",
|
|
88
88
|
"typedoc": "^0.28.20",
|
|
89
|
-
"vitest": "^5.0.
|
|
89
|
+
"vitest": "^5.0.5"
|
|
90
90
|
},
|
|
91
91
|
"engines": {
|
|
92
92
|
"node": ">=22.12"
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The only place phoenix-evals calls the AI SDK's experimental decision API.
|
|
3
|
+
* `experimental_decide` and the decision model spec may change in AI SDK
|
|
4
|
+
* patch releases, so keep everything that depends on their shape in here.
|
|
5
|
+
*/
|
|
6
|
+
import { experimental_decide } from "ai";
|
|
7
|
+
|
|
8
|
+
import { getTelemetryOptions } from "../telemetry";
|
|
9
|
+
import type { ClassificationResult } from "../types/evals";
|
|
10
|
+
import type { TelemetryConfig } from "../types/otel";
|
|
11
|
+
import type { WithPrompt } from "../types/prompts";
|
|
12
|
+
import type { DecisionModel } from "../utils/isDecisionModel";
|
|
13
|
+
import { promptToDecisionState } from "./decisionPrompt";
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The ID of the single choice question asked per classification.
|
|
17
|
+
*/
|
|
18
|
+
const QUESTION_ID = "label";
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* The question's instructions. The rendered prompt, rubric included, is sent
|
|
22
|
+
* as the state.
|
|
23
|
+
*/
|
|
24
|
+
const INSTRUCTIONS =
|
|
25
|
+
"Read the prompt in the state and answer it by selecting exactly one of the choices.";
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Classify with an AI SDK decision model by asking one choice question whose
|
|
29
|
+
* options are the labels. Decision models don't generate text, so the
|
|
30
|
+
* result has no explanation. Label probabilities, the provider's confidence
|
|
31
|
+
* (when it reports one) and the resolved model ID go in the metadata.
|
|
32
|
+
*
|
|
33
|
+
* Choice limits and refusals are left to the AI SDK and the provider: a
|
|
34
|
+
* refusal throws the AI SDK's `Experimental_DecisionRefusalError`.
|
|
35
|
+
*/
|
|
36
|
+
export async function decideClassification(args: {
|
|
37
|
+
model: DecisionModel;
|
|
38
|
+
labels: readonly string[];
|
|
39
|
+
prompt: WithPrompt;
|
|
40
|
+
telemetry?: TelemetryConfig;
|
|
41
|
+
}): Promise<ClassificationResult> {
|
|
42
|
+
const { model, labels, prompt, telemetry } = args;
|
|
43
|
+
const result = await experimental_decide({
|
|
44
|
+
model,
|
|
45
|
+
state: promptToDecisionState(prompt),
|
|
46
|
+
questions: {
|
|
47
|
+
[QUESTION_ID]: {
|
|
48
|
+
type: "choice",
|
|
49
|
+
instructions: INSTRUCTIONS,
|
|
50
|
+
criteria: Object.fromEntries(labels.map((label) => [label, null])),
|
|
51
|
+
},
|
|
52
|
+
},
|
|
53
|
+
telemetry: getTelemetryOptions({
|
|
54
|
+
functionId: "decideClassification",
|
|
55
|
+
telemetry,
|
|
56
|
+
}),
|
|
57
|
+
});
|
|
58
|
+
const answer = result.answers[QUESTION_ID];
|
|
59
|
+
const confidence = getConfidence(result.providerMetadata);
|
|
60
|
+
return {
|
|
61
|
+
label: answer.choice,
|
|
62
|
+
metadata: {
|
|
63
|
+
probabilities: answer.probabilities,
|
|
64
|
+
...(confidence !== undefined ? { confidence } : {}),
|
|
65
|
+
modelId: result.response.modelId,
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The decision model spec has no confidence field, but providers that
|
|
72
|
+
* report one (OpenAI, TypeSafe) put it in their provider metadata as
|
|
73
|
+
* `confidence[questionId]`.
|
|
74
|
+
*/
|
|
75
|
+
function getConfidence(providerMetadata: unknown): number | undefined {
|
|
76
|
+
if (typeof providerMetadata !== "object" || providerMetadata === null) {
|
|
77
|
+
return undefined;
|
|
78
|
+
}
|
|
79
|
+
for (const metadata of Object.values(providerMetadata)) {
|
|
80
|
+
const confidence = (metadata as { confidence?: Record<string, unknown> })
|
|
81
|
+
?.confidence?.[QUESTION_ID];
|
|
82
|
+
if (typeof confidence === "number") return confidence;
|
|
83
|
+
}
|
|
84
|
+
return undefined;
|
|
85
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { ModelMessage } from "ai";
|
|
2
|
+
|
|
3
|
+
import type { WithPrompt } from "../types/prompts";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Join a rendered AI SDK prompt into the plain-text state a decision model
|
|
7
|
+
* judges. Decision models have no roles, so system text, instructions and
|
|
8
|
+
* messages are joined in order.
|
|
9
|
+
*/
|
|
10
|
+
export function promptToDecisionState(prompt: WithPrompt): string {
|
|
11
|
+
const parts: string[] = [];
|
|
12
|
+
const { instructions, system, messages, prompt: promptContent } = prompt;
|
|
13
|
+
for (const value of [instructions, system]) {
|
|
14
|
+
if (value !== undefined) parts.push(instructionsToText(value));
|
|
15
|
+
}
|
|
16
|
+
if (typeof promptContent === "string") {
|
|
17
|
+
parts.push(promptContent);
|
|
18
|
+
} else if (Array.isArray(promptContent)) {
|
|
19
|
+
parts.push(messagesToText(promptContent));
|
|
20
|
+
}
|
|
21
|
+
if (messages !== undefined) parts.push(messagesToText(messages));
|
|
22
|
+
return parts.filter((part) => part.length > 0).join("\n\n");
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function messagesToText(messages: readonly ModelMessage[]): string {
|
|
26
|
+
return messages
|
|
27
|
+
.map((message) => contentToText(message.content))
|
|
28
|
+
.filter((text) => text.length > 0)
|
|
29
|
+
.join("\n\n");
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function instructionsToText(value: unknown): string {
|
|
33
|
+
if (typeof value === "string") return value;
|
|
34
|
+
if (Array.isArray(value)) {
|
|
35
|
+
return value
|
|
36
|
+
.map((item) =>
|
|
37
|
+
typeof item === "object" && item !== null && "content" in item
|
|
38
|
+
? contentToText(item.content)
|
|
39
|
+
: ""
|
|
40
|
+
)
|
|
41
|
+
.filter((text) => text.length > 0)
|
|
42
|
+
.join("\n\n");
|
|
43
|
+
}
|
|
44
|
+
if (typeof value === "object" && value !== null && "content" in value) {
|
|
45
|
+
return contentToText(value.content);
|
|
46
|
+
}
|
|
47
|
+
return "";
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Keep only the text of a message. Decision models judge text here; image
|
|
52
|
+
* and file parts are not forwarded.
|
|
53
|
+
*/
|
|
54
|
+
function contentToText(content: unknown): string {
|
|
55
|
+
if (typeof content === "string") return content;
|
|
56
|
+
if (!Array.isArray(content)) return "";
|
|
57
|
+
return content
|
|
58
|
+
.map((part) =>
|
|
59
|
+
typeof part === "object" &&
|
|
60
|
+
part !== null &&
|
|
61
|
+
"type" in part &&
|
|
62
|
+
part.type === "text" &&
|
|
63
|
+
"text" in part &&
|
|
64
|
+
typeof part.text === "string"
|
|
65
|
+
? part.text
|
|
66
|
+
: ""
|
|
67
|
+
)
|
|
68
|
+
.filter((text) => text.length > 0)
|
|
69
|
+
.join("\n");
|
|
70
|
+
}
|
|
@@ -53,6 +53,27 @@ function resultHasExplanation(
|
|
|
53
53
|
);
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
+
/**
|
|
57
|
+
* Type guard to check if a value has a metadata property.
|
|
58
|
+
*
|
|
59
|
+
* @param result - The value to check
|
|
60
|
+
* @returns True if the value is an object with a metadata record
|
|
61
|
+
*
|
|
62
|
+
* @internal
|
|
63
|
+
*/
|
|
64
|
+
function resultHasMetadata(
|
|
65
|
+
result: unknown
|
|
66
|
+
): result is { metadata: Record<string, unknown> } {
|
|
67
|
+
return (
|
|
68
|
+
typeof result === "object" &&
|
|
69
|
+
result !== null &&
|
|
70
|
+
"metadata" in result &&
|
|
71
|
+
typeof result.metadata === "object" &&
|
|
72
|
+
result.metadata !== null &&
|
|
73
|
+
!Array.isArray(result.metadata)
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
|
|
56
77
|
/**
|
|
57
78
|
* Converts an unknown value to an {@link EvaluationResult}.
|
|
58
79
|
*
|
|
@@ -62,7 +83,7 @@ function resultHasExplanation(
|
|
|
62
83
|
*
|
|
63
84
|
* - **Numbers**: Converted to `{ score: number }`
|
|
64
85
|
* - **Strings**: Converted to `{ label: string }`
|
|
65
|
-
* - **Objects**: Extracts `score`, `label`, and `
|
|
86
|
+
* - **Objects**: Extracts `score`, `label`, `explanation`, and `metadata` properties if present
|
|
66
87
|
* - **Other types**: Returns an empty `EvaluationResult` object
|
|
67
88
|
*
|
|
68
89
|
* This is particularly useful when creating evaluators from functions that may
|
|
@@ -71,7 +92,7 @@ function resultHasExplanation(
|
|
|
71
92
|
* @param result - The value to convert to an EvaluationResult. Can be:
|
|
72
93
|
* - A number (converted to score)
|
|
73
94
|
* - A string (converted to label)
|
|
74
|
-
* - An object with optional `score`, `label`, and/or `
|
|
95
|
+
* - An object with optional `score`, `label`, `explanation`, and/or `metadata` properties
|
|
75
96
|
* - Any other value (returns empty object)
|
|
76
97
|
*
|
|
77
98
|
* @returns An {@link EvaluationResult} object with extracted properties
|
|
@@ -91,7 +112,7 @@ function resultHasExplanation(
|
|
|
91
112
|
* ```
|
|
92
113
|
*
|
|
93
114
|
* @example
|
|
94
|
-
* Convert an object with
|
|
115
|
+
* Convert an object with multiple properties:
|
|
95
116
|
* ```typescript
|
|
96
117
|
* const result = toEvaluationResult({
|
|
97
118
|
* score: 0.9,
|
|
@@ -139,6 +160,9 @@ export function toEvaluationResult(result: unknown): EvaluationResult {
|
|
|
139
160
|
if (resultHasExplanation(result)) {
|
|
140
161
|
evaluationResult.explanation = result.explanation;
|
|
141
162
|
}
|
|
163
|
+
if (resultHasMetadata(result)) {
|
|
164
|
+
evaluationResult.metadata = result.metadata;
|
|
165
|
+
}
|
|
142
166
|
return evaluationResult;
|
|
143
167
|
}
|
|
144
168
|
return {};
|
|
@@ -9,7 +9,7 @@ import type {
|
|
|
9
9
|
WithPromptTemplate,
|
|
10
10
|
} from "../types";
|
|
11
11
|
import type { ObjectMapping } from "../types/data";
|
|
12
|
-
import type {
|
|
12
|
+
import type { DecisionModel } from "../utils/isDecisionModel";
|
|
13
13
|
import { remapObject } from "../utils/objectMappingUtils";
|
|
14
14
|
import { createClassifierFn } from "./createClassifierFn";
|
|
15
15
|
import { LLMEvaluator } from "./LLMEvaluator";
|
|
@@ -30,7 +30,7 @@ export class ClassificationEvaluator<RecordType extends Record<string, unknown>>
|
|
|
30
30
|
/**
|
|
31
31
|
* The model to use for classification
|
|
32
32
|
*/
|
|
33
|
-
readonly model: LanguageModel |
|
|
33
|
+
readonly model: LanguageModel | DecisionModel;
|
|
34
34
|
/**
|
|
35
35
|
* The choices to classify the example into
|
|
36
36
|
*/
|
|
@@ -1,14 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { generateObject } from "ai";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
|
|
4
|
-
import {
|
|
4
|
+
import { decideClassification } from "../decision/decideClassification";
|
|
5
|
+
import { getTelemetryOptions } from "../telemetry";
|
|
5
6
|
import type { ClassificationResult, WithLLM } from "../types/evals";
|
|
6
7
|
import type { WithTelemetry } from "../types/otel";
|
|
7
8
|
import type { WithPrompt } from "../types/prompts";
|
|
8
|
-
import {
|
|
9
|
-
type EvaluationModel,
|
|
10
|
-
isEvaluationModel,
|
|
11
|
-
} from "../utils/isEvaluationModel";
|
|
9
|
+
import { isDecisionModel } from "../utils/isDecisionModel";
|
|
12
10
|
|
|
13
11
|
export type ClassifyArgs = WithLLM &
|
|
14
12
|
WithTelemetry &
|
|
@@ -29,11 +27,11 @@ export type ClassifyArgs = WithLLM &
|
|
|
29
27
|
/**
|
|
30
28
|
* A function that leverages an llm to perform a classification
|
|
31
29
|
*
|
|
32
|
-
* When given an AI SDK
|
|
33
|
-
* classification is routed through `
|
|
34
|
-
* choice question
|
|
35
|
-
* is returned in that case
|
|
36
|
-
*
|
|
30
|
+
* When given an AI SDK decision model (e.g. `openai.decisionModel("gpt-6-luna")`)
|
|
31
|
+
* the classification is routed through `experimental_decide` as a single
|
|
32
|
+
* choice question, with the prompt sent as plain text. Decision models cannot
|
|
33
|
+
* generate text, so no explanation is returned in that case, and
|
|
34
|
+
* `schemaName` / `schemaDescription` are ignored.
|
|
37
35
|
*/
|
|
38
36
|
export async function generateClassification(
|
|
39
37
|
args: ClassifyArgs
|
|
@@ -41,16 +39,15 @@ export async function generateClassification(
|
|
|
41
39
|
const { labels, model, schemaName, schemaDescription, telemetry, ...prompt } =
|
|
42
40
|
args;
|
|
43
41
|
|
|
44
|
-
if (
|
|
45
|
-
return
|
|
42
|
+
if (isDecisionModel(model)) {
|
|
43
|
+
return decideClassification({
|
|
44
|
+
model,
|
|
45
|
+
labels,
|
|
46
|
+
prompt,
|
|
47
|
+
telemetry,
|
|
48
|
+
});
|
|
46
49
|
}
|
|
47
50
|
|
|
48
|
-
const telemetryOptions = {
|
|
49
|
-
isEnabled: telemetry?.isEnabled ?? true,
|
|
50
|
-
functionId: "generateClassification",
|
|
51
|
-
integrations: getTelemetryIntegrations(telemetry?.tracer ?? tracer),
|
|
52
|
-
};
|
|
53
|
-
|
|
54
51
|
const result = await generateObject({
|
|
55
52
|
model,
|
|
56
53
|
schemaName,
|
|
@@ -59,7 +56,10 @@ export async function generateClassification(
|
|
|
59
56
|
explanation: z.string(), // We place the explanation in hopes it uses reasoning to explain the label.
|
|
60
57
|
label: z.enum(labels),
|
|
61
58
|
}),
|
|
62
|
-
telemetry:
|
|
59
|
+
telemetry: getTelemetryOptions({
|
|
60
|
+
functionId: "generateClassification",
|
|
61
|
+
telemetry,
|
|
62
|
+
}),
|
|
63
63
|
// AI SDK 7 rejects system messages inside `messages` by default; keep
|
|
64
64
|
// accepting them since prompt templates may include system messages.
|
|
65
65
|
allowSystemInMessages: true,
|
|
@@ -70,65 +70,3 @@ export async function generateClassification(
|
|
|
70
70
|
explanation: result.object.explanation,
|
|
71
71
|
};
|
|
72
72
|
}
|
|
73
|
-
|
|
74
|
-
type EvaluationState = Parameters<typeof experimental_evaluate>[0]["state"];
|
|
75
|
-
|
|
76
|
-
/**
|
|
77
|
-
* Classify using an AI SDK evaluation model. The rendered prompt becomes the
|
|
78
|
-
* shared evaluation state and the labels become the options of one choice
|
|
79
|
-
* question. The model's label probabilities and resolved model id are
|
|
80
|
-
* returned as metadata.
|
|
81
|
-
*/
|
|
82
|
-
async function evaluateClassification(args: {
|
|
83
|
-
model: EvaluationModel;
|
|
84
|
-
labels: [string, ...string[]];
|
|
85
|
-
prompt: WithPrompt;
|
|
86
|
-
}): Promise<ClassificationResult> {
|
|
87
|
-
const { model, labels, prompt } = args;
|
|
88
|
-
const result = await experimental_evaluate({
|
|
89
|
-
model,
|
|
90
|
-
state: toEvaluationState(prompt),
|
|
91
|
-
questions: {
|
|
92
|
-
label: {
|
|
93
|
-
type: "choice",
|
|
94
|
-
instructions:
|
|
95
|
-
"Read the prompt in the state and answer it by selecting exactly one of the choices.",
|
|
96
|
-
criteria: Object.fromEntries(labels.map((label) => [label, null])),
|
|
97
|
-
},
|
|
98
|
-
},
|
|
99
|
-
});
|
|
100
|
-
const answer = result.answers.label;
|
|
101
|
-
return {
|
|
102
|
-
label: answer.choice,
|
|
103
|
-
metadata: {
|
|
104
|
-
probabilities: answer.probabilities,
|
|
105
|
-
modelId: result.response.modelId,
|
|
106
|
-
},
|
|
107
|
-
};
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
/**
|
|
111
|
-
* Convert an AI SDK prompt into JSON-compatible evaluation state.
|
|
112
|
-
* A bare text prompt is passed through as a string; anything else
|
|
113
|
-
* (instructions, system messages, message arrays) is serialized to JSON.
|
|
114
|
-
*/
|
|
115
|
-
function toEvaluationState(prompt: WithPrompt): EvaluationState {
|
|
116
|
-
const { instructions, system, messages, prompt: promptContent } = prompt;
|
|
117
|
-
if (
|
|
118
|
-
typeof promptContent === "string" &&
|
|
119
|
-
instructions === undefined &&
|
|
120
|
-
system === undefined &&
|
|
121
|
-
messages === undefined
|
|
122
|
-
) {
|
|
123
|
-
return promptContent;
|
|
124
|
-
}
|
|
125
|
-
const state: Record<string, unknown> = {};
|
|
126
|
-
if (instructions !== undefined) state.instructions = instructions;
|
|
127
|
-
if (system !== undefined) state.system = system;
|
|
128
|
-
if (promptContent !== undefined) state.prompt = promptContent;
|
|
129
|
-
if (messages !== undefined) state.messages = messages;
|
|
130
|
-
// Round-trip through JSON so the state only contains JSON values
|
|
131
|
-
// (strips `undefined`, turns Date/URL into strings). Binary file parts
|
|
132
|
-
// are not supported by evaluation models and are not stripped here.
|
|
133
|
-
return JSON.parse(JSON.stringify(state)) as EvaluationState;
|
|
134
|
-
}
|
package/src/telemetry/index.ts
CHANGED
|
@@ -2,6 +2,8 @@ import { OpenTelemetry } from "@ai-sdk/otel";
|
|
|
2
2
|
import { trace, type Tracer } from "@opentelemetry/api";
|
|
3
3
|
import type { Telemetry } from "ai";
|
|
4
4
|
|
|
5
|
+
import type { TelemetryConfig } from "../types/otel";
|
|
6
|
+
|
|
5
7
|
const DEFAULT_TRACER_NAME = "phoenix-evals";
|
|
6
8
|
|
|
7
9
|
/**
|
|
@@ -55,6 +57,25 @@ export function getTelemetryIntegrations(
|
|
|
55
57
|
usage: true,
|
|
56
58
|
providerMetadata: true,
|
|
57
59
|
schema: true,
|
|
60
|
+
// Record the state, questions and answers of decision model calls.
|
|
61
|
+
experimental_decision: true,
|
|
58
62
|
}),
|
|
59
63
|
];
|
|
60
64
|
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Builds the AI SDK `telemetry` call option for an evaluator call. Telemetry
|
|
68
|
+
* is on unless explicitly disabled, and spans go to the configured tracer or
|
|
69
|
+
* the default Phoenix tracer.
|
|
70
|
+
*/
|
|
71
|
+
export function getTelemetryOptions(args: {
|
|
72
|
+
functionId: string;
|
|
73
|
+
telemetry?: TelemetryConfig;
|
|
74
|
+
}) {
|
|
75
|
+
const { functionId, telemetry } = args;
|
|
76
|
+
return {
|
|
77
|
+
isEnabled: telemetry?.isEnabled ?? true,
|
|
78
|
+
functionId,
|
|
79
|
+
integrations: getTelemetryIntegrations(telemetry?.tracer ?? tracer),
|
|
80
|
+
};
|
|
81
|
+
}
|
package/src/types/evals.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { LanguageModel } from "ai";
|
|
2
2
|
|
|
3
|
-
import type {
|
|
3
|
+
import type { DecisionModel } from "../utils/isDecisionModel";
|
|
4
4
|
import type { ObjectMapping } from "./data";
|
|
5
5
|
import type { WithTelemetry } from "./otel";
|
|
6
6
|
import type { PromptTemplate } from "./templating";
|
|
@@ -18,10 +18,12 @@ export interface ExampleRecord<OutputType, InputType> {
|
|
|
18
18
|
export interface WithLLM {
|
|
19
19
|
/**
|
|
20
20
|
* Either a regular AI SDK {@link LanguageModel}, or an AI SDK
|
|
21
|
-
* {@link
|
|
22
|
-
* generating text and therefore cannot produce an
|
|
21
|
+
* {@link DecisionModel} (e.g. `openai.decisionModel("gpt-6-luna")`) which
|
|
22
|
+
* classifies without generating text and therefore cannot produce an
|
|
23
|
+
* explanation. With a decision model, label probabilities are returned in
|
|
24
|
+
* the result's `metadata.probabilities`.
|
|
23
25
|
*/
|
|
24
|
-
model: LanguageModel |
|
|
26
|
+
model: LanguageModel | DecisionModel;
|
|
25
27
|
}
|
|
26
28
|
|
|
27
29
|
// eslint-disable-next-line @typescript-eslint/no-empty-object-type
|
|
@@ -48,7 +50,7 @@ export interface EvaluationResult {
|
|
|
48
50
|
explanation?: string;
|
|
49
51
|
/**
|
|
50
52
|
* Additional information about how the result was produced, e.g. the
|
|
51
|
-
* label probabilities reported by
|
|
53
|
+
* label probabilities reported by a decision model.
|
|
52
54
|
*/
|
|
53
55
|
metadata?: Record<string, unknown>;
|
|
54
56
|
}
|
|
@@ -82,7 +84,7 @@ export interface CreateClassifierArgs extends WithTelemetry {
|
|
|
82
84
|
/*
|
|
83
85
|
* The model to use for classification / evaluation
|
|
84
86
|
*/
|
|
85
|
-
model: LanguageModel |
|
|
87
|
+
model: LanguageModel | DecisionModel;
|
|
86
88
|
/**
|
|
87
89
|
* The choices to classify the example into.
|
|
88
90
|
* e.g. { "correct": 1, "incorrect": 0 }
|
package/src/utils/index.ts
CHANGED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { Experimental_DecisionModel, LanguageModel } from "ai";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* An AI SDK decision model instance, e.g. `openai.decisionModel("gpt-6-luna")`
|
|
5
|
+
* or TypeSafe's Jev. Unlike a {@link LanguageModel}, a decision model only
|
|
6
|
+
* answers typed questions (here, which label applies) and cannot generate
|
|
7
|
+
* free-form text, so classifications made with it have no explanation.
|
|
8
|
+
*
|
|
9
|
+
* Gateway string IDs are excluded: a plain string is treated as a language
|
|
10
|
+
* model. Wrap the ID with the provider's `decisionModel()` instead.
|
|
11
|
+
*/
|
|
12
|
+
export type DecisionModel = Exclude<Experimental_DecisionModel, string>;
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Type guard that distinguishes an AI SDK decision model from a language model.
|
|
16
|
+
* Decision models expose `doDecide` (or `doEvaluate` for providers written
|
|
17
|
+
* against the older evaluation model spec) instead of `doGenerate`.
|
|
18
|
+
*/
|
|
19
|
+
export function isDecisionModel(
|
|
20
|
+
model: LanguageModel | DecisionModel
|
|
21
|
+
): model is DecisionModel {
|
|
22
|
+
if (typeof model !== "object" || model === null) {
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
return (
|
|
26
|
+
("doDecide" in model && typeof model.doDecide === "function") ||
|
|
27
|
+
("doEvaluate" in model && typeof model.doEvaluate === "function")
|
|
28
|
+
);
|
|
29
|
+
}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import type { Experimental_EvaluationModel, LanguageModel } from "ai";
|
|
2
|
-
/**
|
|
3
|
-
* An AI SDK evaluation model instance (e.g. TypeSafe's Jev). Unlike a
|
|
4
|
-
* {@link LanguageModel}, evaluation models only answer typed questions and
|
|
5
|
-
* cannot generate free-form text.
|
|
6
|
-
*/
|
|
7
|
-
export type EvaluationModel = Exclude<Experimental_EvaluationModel, string>;
|
|
8
|
-
/**
|
|
9
|
-
* Type guard that distinguishes an AI SDK evaluation model from a language model.
|
|
10
|
-
* Evaluation models expose `doEvaluate` instead of `doGenerate`.
|
|
11
|
-
*/
|
|
12
|
-
export declare function isEvaluationModel(model: LanguageModel | EvaluationModel): model is EvaluationModel;
|
|
13
|
-
//# sourceMappingURL=isEvaluationModel.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"isEvaluationModel.d.ts","sourceRoot":"","sources":["../../../src/utils/isEvaluationModel.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,4BAA4B,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEtE;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,4BAA4B,EAAE,MAAM,CAAC,CAAC;AAE5E;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,aAAa,GAAG,eAAe,GACrC,KAAK,IAAI,eAAe,CAO1B"}
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Type guard that distinguishes an AI SDK evaluation model from a language model.
|
|
3
|
-
* Evaluation models expose `doEvaluate` instead of `doGenerate`.
|
|
4
|
-
*/
|
|
5
|
-
export function isEvaluationModel(model) {
|
|
6
|
-
return (typeof model === "object" &&
|
|
7
|
-
model !== null &&
|
|
8
|
-
"doEvaluate" in model &&
|
|
9
|
-
typeof model.doEvaluate === "function");
|
|
10
|
-
}
|
|
11
|
-
//# sourceMappingURL=isEvaluationModel.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"isEvaluationModel.js","sourceRoot":"","sources":["../../../src/utils/isEvaluationModel.ts"],"names":[],"mappings":"AASA;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAsC;IAEtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,IAAI,KAAK;QACrB,OAAO,KAAK,CAAC,UAAU,KAAK,UAAU,CACvC,CAAC;AACJ,CAAC"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import type { Experimental_EvaluationModel, LanguageModel } from "ai";
|
|
2
|
-
/**
|
|
3
|
-
* An AI SDK evaluation model instance (e.g. TypeSafe's Jev). Unlike a
|
|
4
|
-
* {@link LanguageModel}, evaluation models only answer typed questions and
|
|
5
|
-
* cannot generate free-form text.
|
|
6
|
-
*/
|
|
7
|
-
export type EvaluationModel = Exclude<Experimental_EvaluationModel, string>;
|
|
8
|
-
/**
|
|
9
|
-
* Type guard that distinguishes an AI SDK evaluation model from a language model.
|
|
10
|
-
* Evaluation models expose `doEvaluate` instead of `doGenerate`.
|
|
11
|
-
*/
|
|
12
|
-
export declare function isEvaluationModel(model: LanguageModel | EvaluationModel): model is EvaluationModel;
|
|
13
|
-
//# sourceMappingURL=isEvaluationModel.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"isEvaluationModel.d.ts","sourceRoot":"","sources":["../../../src/utils/isEvaluationModel.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,4BAA4B,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAEtE;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,4BAA4B,EAAE,MAAM,CAAC,CAAC;AAE5E;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,aAAa,GAAG,eAAe,GACrC,KAAK,IAAI,eAAe,CAO1B"}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.isEvaluationModel = isEvaluationModel;
|
|
4
|
-
/**
|
|
5
|
-
* Type guard that distinguishes an AI SDK evaluation model from a language model.
|
|
6
|
-
* Evaluation models expose `doEvaluate` instead of `doGenerate`.
|
|
7
|
-
*/
|
|
8
|
-
function isEvaluationModel(model) {
|
|
9
|
-
return (typeof model === "object" &&
|
|
10
|
-
model !== null &&
|
|
11
|
-
"doEvaluate" in model &&
|
|
12
|
-
typeof model.doEvaluate === "function");
|
|
13
|
-
}
|
|
14
|
-
//# sourceMappingURL=isEvaluationModel.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"isEvaluationModel.js","sourceRoot":"","sources":["../../../src/utils/isEvaluationModel.ts"],"names":[],"mappings":";;;AASA;;;GAGG;AACH,2BACE,KAAsC;IAEtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,YAAY,IAAI,KAAK;QACrB,OAAO,KAAK,CAAC,UAAU,KAAK,UAAU,CACvC,CAAC;AACJ,CAAC"}
|