@arizeai/phoenix-evals 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/esm/llm/ClassificationEvaluator.d.ts +2 -1
- package/dist/esm/llm/ClassificationEvaluator.d.ts.map +1 -1
- package/dist/esm/llm/ClassificationEvaluator.js.map +1 -1
- package/dist/esm/llm/generateClassification.d.ts +6 -0
- package/dist/esm/llm/generateClassification.d.ts.map +1 -1
- package/dist/esm/llm/generateClassification.js +66 -1
- package/dist/esm/llm/generateClassification.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/esm/types/evals.d.ts +14 -2
- package/dist/esm/types/evals.d.ts.map +1 -1
- package/dist/esm/utils/isEvaluationModel.d.ts +13 -0
- package/dist/esm/utils/isEvaluationModel.d.ts.map +1 -0
- package/dist/esm/utils/isEvaluationModel.js +11 -0
- package/dist/esm/utils/isEvaluationModel.js.map +1 -0
- package/dist/src/llm/ClassificationEvaluator.d.ts +2 -1
- package/dist/src/llm/ClassificationEvaluator.d.ts.map +1 -1
- package/dist/src/llm/ClassificationEvaluator.js.map +1 -1
- package/dist/src/llm/generateClassification.d.ts +6 -0
- package/dist/src/llm/generateClassification.d.ts.map +1 -1
- package/dist/src/llm/generateClassification.js +65 -0
- package/dist/src/llm/generateClassification.js.map +1 -1
- package/dist/src/types/evals.d.ts +14 -2
- package/dist/src/types/evals.d.ts.map +1 -1
- package/dist/src/utils/isEvaluationModel.d.ts +13 -0
- package/dist/src/utils/isEvaluationModel.d.ts.map +1 -0
- package/dist/src/utils/isEvaluationModel.js +14 -0
- package/dist/src/utils/isEvaluationModel.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +11 -10
- package/src/llm/ClassificationEvaluator.ts +2 -1
- package/src/llm/generateClassification.ts +78 -1
- package/src/types/evals.ts +15 -3
- package/src/utils/isEvaluationModel.ts +23 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@arizeai/phoenix-evals",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.6.0",
|
|
4
4
|
"description": "A library for running evaluations for AI use cases",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"arize",
|
|
@@ -66,26 +66,27 @@
|
|
|
66
66
|
}
|
|
67
67
|
},
|
|
68
68
|
"dependencies": {
|
|
69
|
-
"@ai-sdk/otel": "^1.0.
|
|
70
|
-
"@arizeai/openinference-core": "^2.
|
|
69
|
+
"@ai-sdk/otel": "^1.0.107",
|
|
70
|
+
"@arizeai/openinference-core": "^2.7.1",
|
|
71
71
|
"@opentelemetry/api": "^1.9.1",
|
|
72
|
-
"ai": "^7.0.
|
|
72
|
+
"ai": "^7.0.107",
|
|
73
73
|
"jsonpath-plus": "^10.4.0",
|
|
74
74
|
"mustache": "^4.2.0",
|
|
75
|
-
"zod": "^4.5
|
|
75
|
+
"zod": "^4.6.5"
|
|
76
76
|
},
|
|
77
77
|
"devDependencies": {
|
|
78
|
-
"@ai-sdk/google": "^4.0.
|
|
79
|
-
"@ai-sdk/openai": "^4.0.
|
|
80
|
-
"@
|
|
78
|
+
"@ai-sdk/google": "^4.0.76",
|
|
79
|
+
"@ai-sdk/openai": "^4.0.71",
|
|
80
|
+
"@ai-sdk/typesafe-ai": "^3.0.4",
|
|
81
|
+
"@arizeai/openinference-instrumentation-openai": "^4.2.7",
|
|
81
82
|
"@opentelemetry/sdk-trace-node": "^2.11.0",
|
|
82
83
|
"@types/mustache": "^4.2.6",
|
|
83
|
-
"@types/node": "^26.
|
|
84
|
+
"@types/node": "^26.6.2",
|
|
84
85
|
"msw": "^2.15.0",
|
|
85
86
|
"nock": "^14.0.17",
|
|
86
87
|
"tsx": "^4.23.13",
|
|
87
88
|
"typedoc": "^0.28.20",
|
|
88
|
-
"vitest": "^5.0.
|
|
89
|
+
"vitest": "^5.0.1"
|
|
89
90
|
},
|
|
90
91
|
"engines": {
|
|
91
92
|
"node": ">=22.12"
|
|
@@ -9,6 +9,7 @@ import type {
|
|
|
9
9
|
WithPromptTemplate,
|
|
10
10
|
} from "../types";
|
|
11
11
|
import type { ObjectMapping } from "../types/data";
|
|
12
|
+
import type { EvaluationModel } from "../utils/isEvaluationModel";
|
|
12
13
|
import { remapObject } from "../utils/objectMappingUtils";
|
|
13
14
|
import { createClassifierFn } from "./createClassifierFn";
|
|
14
15
|
import { LLMEvaluator } from "./LLMEvaluator";
|
|
@@ -29,7 +30,7 @@ export class ClassificationEvaluator<RecordType extends Record<string, unknown>>
|
|
|
29
30
|
/**
|
|
30
31
|
* The model to use for classification
|
|
31
32
|
*/
|
|
32
|
-
readonly model: LanguageModel;
|
|
33
|
+
readonly model: LanguageModel | EvaluationModel;
|
|
33
34
|
/**
|
|
34
35
|
* The choices to classify the example into
|
|
35
36
|
*/
|
|
@@ -1,10 +1,15 @@
|
|
|
1
|
-
import { generateObject } from "ai";
|
|
1
|
+
import { experimental_evaluate, generateObject } from "ai";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
|
|
4
4
|
import { getTelemetryIntegrations, tracer } from "../telemetry";
|
|
5
5
|
import type { ClassificationResult, WithLLM } from "../types/evals";
|
|
6
6
|
import type { WithTelemetry } from "../types/otel";
|
|
7
7
|
import type { WithPrompt } from "../types/prompts";
|
|
8
|
+
import {
|
|
9
|
+
type EvaluationModel,
|
|
10
|
+
isEvaluationModel,
|
|
11
|
+
} from "../utils/isEvaluationModel";
|
|
12
|
+
|
|
8
13
|
export type ClassifyArgs = WithLLM &
|
|
9
14
|
WithTelemetry &
|
|
10
15
|
WithPrompt & {
|
|
@@ -23,6 +28,12 @@ export type ClassifyArgs = WithLLM &
|
|
|
23
28
|
};
|
|
24
29
|
/**
|
|
25
30
|
* A function that leverages an llm to perform a classification
|
|
31
|
+
*
|
|
32
|
+
* When given an AI SDK evaluation model (e.g. TypeSafe's Jev) the
|
|
33
|
+
* classification is routed through `experimental_evaluate` as a single
|
|
34
|
+
* choice question. Evaluation models cannot generate text, so no explanation
|
|
35
|
+
* is returned in that case. `telemetry`, `schemaName` and `schemaDescription`
|
|
36
|
+
* are ignored on that path: `experimental_evaluate` does not emit spans yet.
|
|
26
37
|
*/
|
|
27
38
|
export async function generateClassification(
|
|
28
39
|
args: ClassifyArgs
|
|
@@ -30,6 +41,10 @@ export async function generateClassification(
|
|
|
30
41
|
const { labels, model, schemaName, schemaDescription, telemetry, ...prompt } =
|
|
31
42
|
args;
|
|
32
43
|
|
|
44
|
+
if (isEvaluationModel(model)) {
|
|
45
|
+
return evaluateClassification({ model, labels, prompt });
|
|
46
|
+
}
|
|
47
|
+
|
|
33
48
|
const telemetryOptions = {
|
|
34
49
|
isEnabled: telemetry?.isEnabled ?? true,
|
|
35
50
|
functionId: "generateClassification",
|
|
@@ -55,3 +70,65 @@ export async function generateClassification(
|
|
|
55
70
|
explanation: result.object.explanation,
|
|
56
71
|
};
|
|
57
72
|
}
|
|
73
|
+
|
|
74
|
+
type EvaluationState = Parameters<typeof experimental_evaluate>[0]["state"];
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Classify using an AI SDK evaluation model. The rendered prompt becomes the
|
|
78
|
+
* shared evaluation state and the labels become the options of one choice
|
|
79
|
+
* question. The model's label probabilities and resolved model id are
|
|
80
|
+
* returned as metadata.
|
|
81
|
+
*/
|
|
82
|
+
async function evaluateClassification(args: {
|
|
83
|
+
model: EvaluationModel;
|
|
84
|
+
labels: [string, ...string[]];
|
|
85
|
+
prompt: WithPrompt;
|
|
86
|
+
}): Promise<ClassificationResult> {
|
|
87
|
+
const { model, labels, prompt } = args;
|
|
88
|
+
const result = await experimental_evaluate({
|
|
89
|
+
model,
|
|
90
|
+
state: toEvaluationState(prompt),
|
|
91
|
+
questions: {
|
|
92
|
+
label: {
|
|
93
|
+
type: "choice",
|
|
94
|
+
instructions:
|
|
95
|
+
"Read the prompt in the state and answer it by selecting exactly one of the choices.",
|
|
96
|
+
criteria: Object.fromEntries(labels.map((label) => [label, null])),
|
|
97
|
+
},
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
const answer = result.answers.label;
|
|
101
|
+
return {
|
|
102
|
+
label: answer.choice,
|
|
103
|
+
metadata: {
|
|
104
|
+
probabilities: answer.probabilities,
|
|
105
|
+
modelId: result.response.modelId,
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Convert an AI SDK prompt into JSON-compatible evaluation state.
|
|
112
|
+
* A bare text prompt is passed through as a string; anything else
|
|
113
|
+
* (instructions, system messages, message arrays) is serialized to JSON.
|
|
114
|
+
*/
|
|
115
|
+
function toEvaluationState(prompt: WithPrompt): EvaluationState {
|
|
116
|
+
const { instructions, system, messages, prompt: promptContent } = prompt;
|
|
117
|
+
if (
|
|
118
|
+
typeof promptContent === "string" &&
|
|
119
|
+
instructions === undefined &&
|
|
120
|
+
system === undefined &&
|
|
121
|
+
messages === undefined
|
|
122
|
+
) {
|
|
123
|
+
return promptContent;
|
|
124
|
+
}
|
|
125
|
+
const state: Record<string, unknown> = {};
|
|
126
|
+
if (instructions !== undefined) state.instructions = instructions;
|
|
127
|
+
if (system !== undefined) state.system = system;
|
|
128
|
+
if (promptContent !== undefined) state.prompt = promptContent;
|
|
129
|
+
if (messages !== undefined) state.messages = messages;
|
|
130
|
+
// Round-trip through JSON so the state only contains JSON values
|
|
131
|
+
// (strips `undefined`, turns Date/URL into strings). Binary file parts
|
|
132
|
+
// are not supported by evaluation models and are not stripped here.
|
|
133
|
+
return JSON.parse(JSON.stringify(state)) as EvaluationState;
|
|
134
|
+
}
|
package/src/types/evals.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { LanguageModel } from "ai";
|
|
2
2
|
|
|
3
|
+
import type { EvaluationModel } from "../utils/isEvaluationModel";
|
|
3
4
|
import type { ObjectMapping } from "./data";
|
|
4
5
|
import type { WithTelemetry } from "./otel";
|
|
5
6
|
import type { PromptTemplate } from "./templating";
|
|
@@ -15,7 +16,12 @@ export interface ExampleRecord<OutputType, InputType> {
|
|
|
15
16
|
}
|
|
16
17
|
|
|
17
18
|
export interface WithLLM {
|
|
18
|
-
|
|
19
|
+
/**
|
|
20
|
+
* Either a regular AI SDK {@link LanguageModel}, or an AI SDK
|
|
21
|
+
* {@link EvaluationModel} (e.g. TypeSafe's Jev) which classifies without
|
|
22
|
+
* generating text and therefore cannot produce an explanation.
|
|
23
|
+
*/
|
|
24
|
+
model: LanguageModel | EvaluationModel;
|
|
19
25
|
}
|
|
20
26
|
|
|
21
27
|
// eslint-disable-next-line @typescript-eslint/no-empty-object-type
|
|
@@ -40,6 +46,11 @@ export interface EvaluationResult {
|
|
|
40
46
|
* @example "The model correctly identified the sentiment of the text."
|
|
41
47
|
*/
|
|
42
48
|
explanation?: string;
|
|
49
|
+
/**
|
|
50
|
+
* Additional information about how the result was produced, e.g. the
|
|
51
|
+
* label probabilities reported by an evaluation model.
|
|
52
|
+
*/
|
|
53
|
+
metadata?: Record<string, unknown>;
|
|
43
54
|
}
|
|
44
55
|
|
|
45
56
|
/**
|
|
@@ -48,6 +59,7 @@ export interface EvaluationResult {
|
|
|
48
59
|
export interface ClassificationResult {
|
|
49
60
|
label: string;
|
|
50
61
|
explanation?: string;
|
|
62
|
+
metadata?: Record<string, unknown>;
|
|
51
63
|
}
|
|
52
64
|
|
|
53
65
|
/**
|
|
@@ -68,9 +80,9 @@ export type ClassificationChoicesMap = Record<string, number>;
|
|
|
68
80
|
*/
|
|
69
81
|
export interface CreateClassifierArgs extends WithTelemetry {
|
|
70
82
|
/*
|
|
71
|
-
* The
|
|
83
|
+
* The model to use for classification / evaluation
|
|
72
84
|
*/
|
|
73
|
-
model: LanguageModel;
|
|
85
|
+
model: LanguageModel | EvaluationModel;
|
|
74
86
|
/**
|
|
75
87
|
* The choices to classify the example into.
|
|
76
88
|
* e.g. { "correct": 1, "incorrect": 0 }
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { Experimental_EvaluationModel, LanguageModel } from "ai";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* An AI SDK evaluation model instance (e.g. TypeSafe's Jev). Unlike a
|
|
5
|
+
* {@link LanguageModel}, evaluation models only answer typed questions and
|
|
6
|
+
* cannot generate free-form text.
|
|
7
|
+
*/
|
|
8
|
+
export type EvaluationModel = Exclude<Experimental_EvaluationModel, string>;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Type guard that distinguishes an AI SDK evaluation model from a language model.
|
|
12
|
+
* Evaluation models expose `doEvaluate` instead of `doGenerate`.
|
|
13
|
+
*/
|
|
14
|
+
export function isEvaluationModel(
|
|
15
|
+
model: LanguageModel | EvaluationModel
|
|
16
|
+
): model is EvaluationModel {
|
|
17
|
+
return (
|
|
18
|
+
typeof model === "object" &&
|
|
19
|
+
model !== null &&
|
|
20
|
+
"doEvaluate" in model &&
|
|
21
|
+
typeof model.doEvaluate === "function"
|
|
22
|
+
);
|
|
23
|
+
}
|