ai 7.0.101 → 7.0.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -0
- package/dist/index.d.ts +138 -22
- package/dist/index.js +2373 -316
- package/dist/index.js.map +1 -1
- package/dist/internal/index.js +2 -1
- package/dist/internal/index.js.map +1 -1
- package/dist/test/index.d.ts +16 -2
- package/dist/test/index.js +17 -0
- package/dist/test/index.js.map +1 -1
- package/docs/03-agents/07-workflow-agent.mdx +1 -1
- package/docs/03-ai-sdk-core/18-code-mode.mdx +39 -0
- package/docs/03-ai-sdk-core/32-evaluation.mdx +110 -0
- package/docs/03-ai-sdk-core/36-realtime.mdx +41 -18
- package/docs/03-ai-sdk-core/42-batch.mdx +1 -2
- package/docs/04-ai-sdk-ui/03-chatbot-message-persistence.mdx +35 -0
- package/docs/04-ai-sdk-ui/21-transport.mdx +1 -1
- package/docs/06-advanced/11-secure-url-fetching.mdx +8 -2
- package/docs/07-reference/01-ai-sdk-core/14-evaluate.mdx +54 -0
- package/docs/07-reference/01-ai-sdk-core/32-validate-ui-messages.mdx +12 -0
- package/docs/07-reference/01-ai-sdk-core/33-safe-validate-ui-messages.mdx +12 -0
- package/docs/07-reference/02-ai-sdk-ui/05-use-realtime.mdx +364 -48
- package/docs/07-reference/02-ai-sdk-ui/31-convert-to-model-messages.mdx +12 -0
- package/docs/07-reference/04-ai-sdk-workflow/02-workflow-chat-transport.mdx +4 -4
- package/docs/07-reference/05-ai-sdk-errors/ai-evaluation-unsupported-question-type-error.mdx +31 -0
- package/docs/07-reference/05-ai-sdk-errors/index.mdx +131 -36
- package/package.json +12 -12
- package/src/error/index.ts +1 -0
- package/src/evaluate/evaluate.ts +112 -0
- package/src/evaluate/evaluation-result.ts +39 -0
- package/src/evaluate/index.ts +7 -0
- package/src/evaluate/validate-evaluation.ts +298 -0
- package/src/generate-text/execute-tools-from-stream.ts +7 -0
- package/src/generate-text/generate-text.ts +16 -12
- package/src/generate-text/stream-text.ts +9 -2
- package/src/generate-text/tool-caller-configuration.ts +58 -4
- package/src/index.ts +1 -0
- package/src/realtime/__fixtures__/fake-live-websocket.ts +71 -0
- package/src/realtime/__fixtures__/fake-realtime.ts +36 -0
- package/src/realtime/__fixtures__/fake-webrtc.ts +133 -0
- package/src/realtime/browser-realtime-audio.ts +107 -10
- package/src/realtime/browser-realtime-live-websocket.ts +247 -0
- package/src/realtime/browser-realtime-transport.ts +235 -69
- package/src/realtime/browser-realtime-webrtc.ts +582 -0
- package/src/realtime/encode-realtime-frame.ts +33 -0
- package/src/realtime/index.ts +1 -0
- package/src/realtime/realtime-attempt.ts +45 -0
- package/src/realtime/realtime-command-tracker.ts +81 -0
- package/src/realtime/realtime-event-channel.ts +170 -0
- package/src/realtime/realtime-event-reducer.ts +3 -0
- package/src/realtime/realtime-session-state.ts +65 -0
- package/src/realtime/realtime-session.ts +768 -218
- package/src/realtime/realtime-types.ts +1 -1
- package/src/realtime/validate-realtime-setup.ts +37 -0
- package/src/test/evaluation-mock-model-v4.ts +27 -0
- package/src/ui/convert-to-model-messages.ts +3 -0
- package/src/ui/process-ui-message-stream.ts +4 -0
- package/src/ui/ui-messages.ts +5 -1
- package/src/ui/validate-ui-messages.ts +3 -0
- package/src/ui/warn-if-ui-message-has-deprecated-raw-input.ts +36 -0
|
@@ -13,7 +13,7 @@ Unlike [`DefaultChatTransport`](/docs/ai-sdk-ui/transport) which assumes the ful
|
|
|
13
13
|
'use client';
|
|
14
14
|
|
|
15
15
|
import { useChat } from '@ai-sdk/react';
|
|
16
|
-
import { WorkflowChatTransport } from '@ai-sdk/workflow';
|
|
16
|
+
import { WorkflowChatTransport } from '@ai-sdk/workflow/client';
|
|
17
17
|
|
|
18
18
|
export default function Chat() {
|
|
19
19
|
const { messages, sendMessage } = useChat({
|
|
@@ -30,7 +30,7 @@ export default function Chat() {
|
|
|
30
30
|
## Import
|
|
31
31
|
|
|
32
32
|
<Snippet
|
|
33
|
-
text={`import { WorkflowChatTransport } from "@ai-sdk/workflow"`}
|
|
33
|
+
text={`import { WorkflowChatTransport } from "@ai-sdk/workflow/client"`}
|
|
34
34
|
prompt={false}
|
|
35
35
|
/>
|
|
36
36
|
|
|
@@ -238,7 +238,7 @@ See the [WorkflowAgent guide](/docs/agents/workflow-agent) for complete endpoint
|
|
|
238
238
|
'use client';
|
|
239
239
|
|
|
240
240
|
import { useChat } from '@ai-sdk/react';
|
|
241
|
-
import { WorkflowChatTransport } from '@ai-sdk/workflow';
|
|
241
|
+
import { WorkflowChatTransport } from '@ai-sdk/workflow/client';
|
|
242
242
|
import { useMemo } from 'react';
|
|
243
243
|
|
|
244
244
|
export default function Chat() {
|
|
@@ -271,7 +271,7 @@ export default function Chat() {
|
|
|
271
271
|
'use client';
|
|
272
272
|
|
|
273
273
|
import { useChat } from '@ai-sdk/react';
|
|
274
|
-
import { WorkflowChatTransport } from '@ai-sdk/workflow';
|
|
274
|
+
import { WorkflowChatTransport } from '@ai-sdk/workflow/client';
|
|
275
275
|
import { useMemo } from 'react';
|
|
276
276
|
|
|
277
277
|
export default function Chat() {
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: AI_EvaluationUnsupportedQuestionTypeError
|
|
3
|
+
description: Identify an evaluation question type that a model does not support.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# AI_EvaluationUnsupportedQuestionTypeError
|
|
7
|
+
|
|
8
|
+
This experimental error identifies a question whose type is unsupported by an
|
|
9
|
+
evaluation model. It is exported from `ai` and `@ai-sdk/provider` as
|
|
10
|
+
`Experimental_EvaluationUnsupportedQuestionTypeError`.
|
|
11
|
+
|
|
12
|
+
## Properties
|
|
13
|
+
|
|
14
|
+
- `questionId`: The ID of the unsupported question.
|
|
15
|
+
- `questionType`: The requested question type.
|
|
16
|
+
- `provider`: The provider of the evaluation model.
|
|
17
|
+
- `modelId`: The evaluation model ID.
|
|
18
|
+
- `message`: A description of the unsupported question and model. Providers can
|
|
19
|
+
supply a custom message.
|
|
20
|
+
|
|
21
|
+
## Checking for this Error
|
|
22
|
+
|
|
23
|
+
Use the marker-based `isInstance` check, which works across package copies:
|
|
24
|
+
|
|
25
|
+
```typescript
|
|
26
|
+
import { Experimental_EvaluationUnsupportedQuestionTypeError as EvaluationUnsupportedQuestionTypeError } from 'ai';
|
|
27
|
+
|
|
28
|
+
if (EvaluationUnsupportedQuestionTypeError.isInstance(error)) {
|
|
29
|
+
console.log(error.questionId, error.questionType, error.modelId);
|
|
30
|
+
}
|
|
31
|
+
```
|
|
@@ -1,43 +1,138 @@
|
|
|
1
1
|
---
|
|
2
2
|
title: AI SDK Errors
|
|
3
|
-
description:
|
|
3
|
+
description: Reference for AI SDK error classes and typed error handling.
|
|
4
4
|
collapsed: true
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
# AI SDK Errors
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
9
|
+
The AI SDK exposes typed errors so applications can handle expected failure
|
|
10
|
+
modes without matching error message strings.
|
|
11
|
+
|
|
12
|
+
## Importing Errors
|
|
13
|
+
|
|
14
|
+
The application package is named `ai`, not `@ai-sdk/ai`. It re-exports common
|
|
15
|
+
provider-level errors from `@ai-sdk/provider` together with the higher-level
|
|
16
|
+
errors raised by AI SDK Core:
|
|
17
|
+
|
|
18
|
+
```typescript
|
|
19
|
+
import { APICallError, NoObjectGeneratedError } from 'ai';
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Provider implementations that depend directly on `@ai-sdk/provider` can import
|
|
23
|
+
its provider-level errors from that package instead:
|
|
24
|
+
|
|
25
|
+
```typescript
|
|
26
|
+
import { APICallError, InvalidResponseDataError } from '@ai-sdk/provider';
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## Migrating to Typed Error Handling
|
|
30
|
+
|
|
31
|
+
Replace message matching or an unconditionally generic `catch` block with the
|
|
32
|
+
most specific static `isInstance` guard available. Check `AISDKError` last when
|
|
33
|
+
you need a fallback for any AI SDK error:
|
|
34
|
+
|
|
35
|
+
```typescript
|
|
36
|
+
import { AISDKError, APICallError, generateText } from 'ai';
|
|
37
|
+
|
|
38
|
+
try {
|
|
39
|
+
await generateText({
|
|
40
|
+
model,
|
|
41
|
+
prompt: 'Write a vegetarian lasagna recipe for 4 people.',
|
|
42
|
+
});
|
|
43
|
+
} catch (error) {
|
|
44
|
+
if (APICallError.isInstance(error)) {
|
|
45
|
+
console.error('Provider request failed:', error.statusCode);
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
if (AISDKError.isInstance(error)) {
|
|
50
|
+
console.error('AI SDK error:', error.name);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
throw error;
|
|
55
|
+
}
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Prefer `ErrorClass.isInstance(error)` over `error instanceof ErrorClass` when
|
|
59
|
+
the class provides it. The static guard also works when multiple AI SDK package
|
|
60
|
+
versions are loaded.
|
|
61
|
+
|
|
62
|
+
## Common Failure Modes
|
|
63
|
+
|
|
64
|
+
| Failure mode | Error to check |
|
|
65
|
+
| -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
66
|
+
| A provider request fails because of a network error or non-success response | [`APICallError`](/docs/reference/ai-sdk-errors/ai-api-call-error) |
|
|
67
|
+
| Automatic retries are exhausted | [`RetryError`](/docs/reference/ai-sdk-errors/ai-retry-error) |
|
|
68
|
+
| A provider reports an error after a response stream has started | [`StreamProviderError`](/docs/reference/ai-sdk-errors/ai-stream-provider-error) |
|
|
69
|
+
| A structured output cannot be parsed or validated | [`NoObjectGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-object-generated-error); inspect its `cause` for [`JSONParseError`](/docs/reference/ai-sdk-errors/ai-json-parse-error) or [`TypeValidationError`](/docs/reference/ai-sdk-errors/ai-type-validation-error) |
|
|
70
|
+
| A generation call returns no usable output | [`NoOutputGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-output-generated-error) or the modality-specific `No*GeneratedError` |
|
|
71
|
+
| A model calls a missing tool or supplies invalid tool input | [`NoSuchToolError`](/docs/reference/ai-sdk-errors/ai-no-such-tool-error) or [`InvalidToolInputError`](/docs/reference/ai-sdk-errors/ai-invalid-tool-input-error) |
|
|
72
|
+
| Tool-call repair fails | [`ToolCallRepairError`](/docs/reference/ai-sdk-errors/ai-tool-call-repair-error) |
|
|
73
|
+
| A response violates an enforced `toolChoice` | [`ToolChoiceViolationError`](/docs/reference/ai-sdk-errors/ai-tool-choice-violation-error) |
|
|
74
|
+
| Message history contains unresolved tool calls or invalid approvals | [`MissingToolResultsError`](/docs/troubleshooting/missing-tool-results-error), [`InvalidToolApprovalError`](/docs/reference/ai-sdk-errors/ai-invalid-tool-approval-error), or [`InvalidToolApprovalSignatureError`](/docs/reference/ai-sdk-errors/ai-invalid-tool-approval-signature-error) |
|
|
75
|
+
| A provider or model cannot be resolved | [`NoSuchProviderError`](/docs/reference/ai-sdk-errors/ai-no-such-provider-error), [`NoSuchModelError`](/docs/reference/ai-sdk-errors/ai-no-such-model-error), or [`NoSuchProviderReferenceError`](/docs/reference/ai-sdk-errors/ai-no-such-provider-reference-error) |
|
|
76
|
+
| A provider response is empty, malformed, or invalid | [`EmptyResponseBodyError`](/docs/reference/ai-sdk-errors/ai-empty-response-body-error), [`JSONParseError`](/docs/reference/ai-sdk-errors/ai-json-parse-error), or [`InvalidResponseDataError`](/docs/reference/ai-sdk-errors/ai-invalid-response-data-error) |
|
|
77
|
+
| The requested model or provider does not support a capability or specification version | [`UnsupportedFunctionalityError`](/docs/reference/ai-sdk-errors/ai-unsupported-functionality-error) or [`UnsupportedModelVersionError`](/docs/troubleshooting/unsupported-model-version) |
|
|
78
|
+
| UI messages cannot be converted or a UI message stream is invalid | [`MessageConversionError`](/docs/reference/ai-sdk-errors/ai-message-conversion-error) or [`UIMessageStreamError`](/docs/reference/ai-sdk-errors/ai-ui-message-stream-error) |
|
|
79
|
+
|
|
80
|
+
## Error Reference
|
|
81
|
+
|
|
82
|
+
### Base Error
|
|
83
|
+
|
|
84
|
+
- `AISDKError`: Base class and broad type guard for AI SDK errors.
|
|
85
|
+
|
|
86
|
+
### Provider Requests and Responses
|
|
87
|
+
|
|
88
|
+
- [`APICallError`](/docs/reference/ai-sdk-errors/ai-api-call-error)
|
|
89
|
+
- [`DownloadError`](/docs/reference/ai-sdk-errors/ai-download-error)
|
|
90
|
+
- [`EmptyResponseBodyError`](/docs/reference/ai-sdk-errors/ai-empty-response-body-error)
|
|
91
|
+
- [`InvalidPromptError`](/docs/reference/ai-sdk-errors/ai-invalid-prompt-error)
|
|
92
|
+
- [`InvalidResponseDataError`](/docs/reference/ai-sdk-errors/ai-invalid-response-data-error)
|
|
93
|
+
- [`JSONParseError`](/docs/reference/ai-sdk-errors/ai-json-parse-error)
|
|
94
|
+
- [`LoadAPIKeyError`](/docs/reference/ai-sdk-errors/ai-load-api-key-error)
|
|
95
|
+
- [`LoadSettingError`](/docs/reference/ai-sdk-errors/ai-load-setting-error)
|
|
96
|
+
- [`NoContentGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-content-generated-error)
|
|
97
|
+
- [`NoSuchModelError`](/docs/reference/ai-sdk-errors/ai-no-such-model-error)
|
|
98
|
+
- [`NoSuchProviderReferenceError`](/docs/reference/ai-sdk-errors/ai-no-such-provider-reference-error)
|
|
99
|
+
- [`RetryError`](/docs/reference/ai-sdk-errors/ai-retry-error)
|
|
100
|
+
- [`StreamProviderError`](/docs/reference/ai-sdk-errors/ai-stream-provider-error)
|
|
101
|
+
- [`TooManyEmbeddingValuesForCallError`](/docs/reference/ai-sdk-errors/ai-too-many-embedding-values-for-call-error)
|
|
102
|
+
- [`TypeValidationError`](/docs/reference/ai-sdk-errors/ai-type-validation-error)
|
|
103
|
+
- [`UnsupportedFunctionalityError`](/docs/reference/ai-sdk-errors/ai-unsupported-functionality-error)
|
|
104
|
+
|
|
105
|
+
### Inputs and Message Streams
|
|
106
|
+
|
|
107
|
+
- [`InvalidArgumentError`](/docs/reference/ai-sdk-errors/ai-invalid-argument-error)
|
|
108
|
+
- [`InvalidDataContentError`](/docs/reference/ai-sdk-errors/ai-invalid-data-content-error)
|
|
109
|
+
- [`InvalidMessageRoleError`](/docs/reference/ai-sdk-errors/ai-invalid-message-role-error)
|
|
110
|
+
- `InvalidStreamPartError`
|
|
111
|
+
- [`MessageConversionError`](/docs/reference/ai-sdk-errors/ai-message-conversion-error)
|
|
112
|
+
- [`UIMessageStreamError`](/docs/reference/ai-sdk-errors/ai-ui-message-stream-error)
|
|
113
|
+
|
|
114
|
+
### Generated Output
|
|
115
|
+
|
|
116
|
+
- [`NoImageGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-image-generated-error)
|
|
117
|
+
- [`NoObjectGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-object-generated-error)
|
|
118
|
+
- [`NoOutputGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-output-generated-error)
|
|
119
|
+
- [`NoSpeechGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-speech-generated-error)
|
|
120
|
+
- [`NoTranscriptGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-transcript-generated-error)
|
|
121
|
+
- [`NoTranslationGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-translation-generated-error)
|
|
122
|
+
- [`NoVideoGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-video-generated-error)
|
|
123
|
+
|
|
124
|
+
### Providers and Model Compatibility
|
|
125
|
+
|
|
126
|
+
- [`NoSuchProviderError`](/docs/reference/ai-sdk-errors/ai-no-such-provider-error)
|
|
127
|
+
- [`UnsupportedModelVersionError`](/docs/troubleshooting/unsupported-model-version)
|
|
128
|
+
|
|
129
|
+
### Tools and Approvals
|
|
130
|
+
|
|
131
|
+
- [`InvalidToolApprovalError`](/docs/reference/ai-sdk-errors/ai-invalid-tool-approval-error)
|
|
132
|
+
- [`InvalidToolApprovalSignatureError`](/docs/reference/ai-sdk-errors/ai-invalid-tool-approval-signature-error)
|
|
133
|
+
- [`InvalidToolInputError`](/docs/reference/ai-sdk-errors/ai-invalid-tool-input-error)
|
|
134
|
+
- [`MissingToolResultsError`](/docs/troubleshooting/missing-tool-results-error)
|
|
135
|
+
- [`NoSuchToolError`](/docs/reference/ai-sdk-errors/ai-no-such-tool-error)
|
|
136
|
+
- [`ToolCallNotFoundForApprovalError`](/docs/reference/ai-sdk-errors/ai-tool-call-not-found-for-approval-error)
|
|
137
|
+
- [`ToolCallRepairError`](/docs/reference/ai-sdk-errors/ai-tool-call-repair-error)
|
|
138
|
+
- [`ToolChoiceViolationError`](/docs/reference/ai-sdk-errors/ai-tool-choice-violation-error)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai",
|
|
3
|
-
"version": "7.0.
|
|
3
|
+
"version": "7.0.103",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -42,20 +42,20 @@
|
|
|
42
42
|
}
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
|
-
"@ai-sdk/gateway": "4.0.
|
|
46
|
-
"@ai-sdk/provider": "4.0.
|
|
47
|
-
"@ai-sdk/provider-utils": "5.0.
|
|
45
|
+
"@ai-sdk/gateway": "4.0.83",
|
|
46
|
+
"@ai-sdk/provider": "4.0.16",
|
|
47
|
+
"@ai-sdk/provider-utils": "5.0.42"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
|
-
"@ai-sdk/amazon-bedrock": "5.0.
|
|
51
|
-
"@ai-sdk/deepseek": "3.0.
|
|
52
|
-
"@ai-sdk/google": "4.0.
|
|
53
|
-
"@ai-sdk/groq": "4.0.
|
|
54
|
-
"@ai-sdk/huggingface": "2.0.
|
|
55
|
-
"@ai-sdk/moonshotai": "3.0.
|
|
56
|
-
"@ai-sdk/openai": "4.0.
|
|
50
|
+
"@ai-sdk/amazon-bedrock": "5.0.85",
|
|
51
|
+
"@ai-sdk/deepseek": "3.0.46",
|
|
52
|
+
"@ai-sdk/google": "4.0.73",
|
|
53
|
+
"@ai-sdk/groq": "4.0.43",
|
|
54
|
+
"@ai-sdk/huggingface": "2.0.50",
|
|
55
|
+
"@ai-sdk/moonshotai": "3.0.51",
|
|
56
|
+
"@ai-sdk/openai": "4.0.68",
|
|
57
57
|
"@ai-sdk/test-server": "2.0.1",
|
|
58
|
-
"@ai-sdk/xai": "
|
|
58
|
+
"@ai-sdk/xai": "5.0.1",
|
|
59
59
|
"@edge-runtime/vm": "^5.0.0",
|
|
60
60
|
"@smithy/eventstream-codec": "^4.3.3",
|
|
61
61
|
"@smithy/util-utf8": "^4.3.3",
|
package/src/error/index.ts
CHANGED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import {
|
|
2
|
+
Experimental_EvaluationUnsupportedQuestionTypeError as EvaluationUnsupportedQuestionTypeError,
|
|
3
|
+
type Experimental_EvaluationModelV4CallOptions as EvaluationModelV4CallOptions,
|
|
4
|
+
} from '@ai-sdk/provider';
|
|
5
|
+
import {
|
|
6
|
+
withUserAgentSuffix,
|
|
7
|
+
type ProviderOptions,
|
|
8
|
+
} from '@ai-sdk/provider-utils';
|
|
9
|
+
import { UnsupportedModelVersionError } from '../error/unsupported-model-version-error';
|
|
10
|
+
import { logWarnings } from '../logger/log-warnings';
|
|
11
|
+
import { prepareRetries } from '../util/prepare-retries';
|
|
12
|
+
import { VERSION } from '../version';
|
|
13
|
+
import type {
|
|
14
|
+
EvaluationModel,
|
|
15
|
+
EvaluationQuestion,
|
|
16
|
+
EvaluationResult,
|
|
17
|
+
} from './evaluation-result';
|
|
18
|
+
import {
|
|
19
|
+
validateEvaluationInput,
|
|
20
|
+
validateEvaluationAnswers,
|
|
21
|
+
} from './validate-evaluation';
|
|
22
|
+
|
|
23
|
+
/** Evaluate typed questions against one shared state. Experimental. */
|
|
24
|
+
export async function evaluate<
|
|
25
|
+
const QUESTIONS extends Record<string, EvaluationQuestion>,
|
|
26
|
+
>({
|
|
27
|
+
model,
|
|
28
|
+
state,
|
|
29
|
+
questions,
|
|
30
|
+
maxRetries,
|
|
31
|
+
abortSignal,
|
|
32
|
+
headers,
|
|
33
|
+
providerOptions = {},
|
|
34
|
+
}: {
|
|
35
|
+
/** An evaluation model instance. String model resolution is not yet supported. */
|
|
36
|
+
model: EvaluationModel;
|
|
37
|
+
state: EvaluationModelV4CallOptions['state'];
|
|
38
|
+
questions: QUESTIONS;
|
|
39
|
+
/** Maximum retries for transient provider failures. Defaults to 2. */
|
|
40
|
+
maxRetries?: number;
|
|
41
|
+
abortSignal?: AbortSignal;
|
|
42
|
+
headers?: Record<string, string>;
|
|
43
|
+
providerOptions?: ProviderOptions;
|
|
44
|
+
}): Promise<EvaluationResult<QUESTIONS>> {
|
|
45
|
+
if (model.specificationVersion !== 'v4') {
|
|
46
|
+
throw new UnsupportedModelVersionError({
|
|
47
|
+
version: model.specificationVersion,
|
|
48
|
+
provider: model.provider,
|
|
49
|
+
modelId: model.modelId,
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
validateEvaluationInput({ state, questions });
|
|
54
|
+
|
|
55
|
+
for (const [questionId, question] of Object.entries(questions)) {
|
|
56
|
+
if (!model.supportedQuestionTypes.includes(question.type)) {
|
|
57
|
+
throw new EvaluationUnsupportedQuestionTypeError({
|
|
58
|
+
questionId,
|
|
59
|
+
questionType: question.type,
|
|
60
|
+
provider: model.provider,
|
|
61
|
+
modelId: model.modelId,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const { retry } = prepareRetries({ maxRetries, abortSignal });
|
|
67
|
+
const result = await retry(() => {
|
|
68
|
+
abortSignal?.throwIfAborted();
|
|
69
|
+
return model.doEvaluate({
|
|
70
|
+
state,
|
|
71
|
+
questions,
|
|
72
|
+
abortSignal,
|
|
73
|
+
headers: withUserAgentSuffix(headers ?? {}, `ai/${VERSION}`),
|
|
74
|
+
providerOptions,
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
abortSignal?.throwIfAborted();
|
|
79
|
+
validateEvaluationAnswers({
|
|
80
|
+
questions,
|
|
81
|
+
answers: result.answers,
|
|
82
|
+
rounding: result.rounding,
|
|
83
|
+
});
|
|
84
|
+
logWarnings({
|
|
85
|
+
warnings: result.warnings,
|
|
86
|
+
provider: model.provider,
|
|
87
|
+
model: model.modelId,
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
const inputTokens = result.usage?.inputTokens;
|
|
91
|
+
const outputTokens = result.usage?.outputTokens;
|
|
92
|
+
|
|
93
|
+
return {
|
|
94
|
+
answers: result.answers as EvaluationResult<QUESTIONS>['answers'],
|
|
95
|
+
usage: {
|
|
96
|
+
inputTokens,
|
|
97
|
+
outputTokens,
|
|
98
|
+
totalTokens:
|
|
99
|
+
inputTokens != null && outputTokens != null
|
|
100
|
+
? inputTokens + outputTokens
|
|
101
|
+
: undefined,
|
|
102
|
+
},
|
|
103
|
+
warnings: result.warnings,
|
|
104
|
+
rounding: result.rounding,
|
|
105
|
+
providerMetadata: result.providerMetadata,
|
|
106
|
+
response: {
|
|
107
|
+
...result.response,
|
|
108
|
+
timestamp: result.response?.timestamp ?? new Date(),
|
|
109
|
+
modelId: result.response?.modelId ?? model.modelId,
|
|
110
|
+
},
|
|
111
|
+
};
|
|
112
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
Experimental_EvaluationModelV4 as EvaluationModelV4,
|
|
3
|
+
Experimental_EvaluationModelV4Question as EvaluationModelV4Question,
|
|
4
|
+
Experimental_EvaluationModelV4Result as EvaluationModelV4Result,
|
|
5
|
+
} from '@ai-sdk/provider';
|
|
6
|
+
|
|
7
|
+
export type EvaluationModel = EvaluationModelV4;
|
|
8
|
+
export type EvaluationQuestion = EvaluationModelV4Question;
|
|
9
|
+
|
|
10
|
+
export type EvaluationAnswer<QUESTION extends EvaluationQuestion> =
|
|
11
|
+
QUESTION extends { type: 'choice'; criteria: infer CRITERIA }
|
|
12
|
+
? {
|
|
13
|
+
type: 'choice';
|
|
14
|
+
choice: Extract<keyof CRITERIA, string>;
|
|
15
|
+
probabilities?: Record<Extract<keyof CRITERIA, string>, number>;
|
|
16
|
+
}
|
|
17
|
+
: QUESTION extends { type: 'score' }
|
|
18
|
+
? { type: 'score'; score: number; probabilities?: Record<string, number> }
|
|
19
|
+
: { type: 'boolean'; probability: number };
|
|
20
|
+
|
|
21
|
+
export type EvaluationResult<
|
|
22
|
+
QUESTIONS extends Record<string, EvaluationQuestion>,
|
|
23
|
+
> = {
|
|
24
|
+
readonly answers: {
|
|
25
|
+
[ID in keyof QUESTIONS]: EvaluationAnswer<QUESTIONS[ID]>;
|
|
26
|
+
};
|
|
27
|
+
readonly usage: {
|
|
28
|
+
inputTokens: number | undefined;
|
|
29
|
+
outputTokens: number | undefined;
|
|
30
|
+
totalTokens: number | undefined;
|
|
31
|
+
};
|
|
32
|
+
readonly warnings: EvaluationModelV4Result['warnings'];
|
|
33
|
+
readonly rounding: EvaluationModelV4Result['rounding'];
|
|
34
|
+
readonly providerMetadata: EvaluationModelV4Result['providerMetadata'];
|
|
35
|
+
readonly response: NonNullable<EvaluationModelV4Result['response']> & {
|
|
36
|
+
timestamp: Date;
|
|
37
|
+
modelId: string;
|
|
38
|
+
};
|
|
39
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { evaluate as experimental_evaluate } from './evaluate';
|
|
2
|
+
export type {
|
|
3
|
+
EvaluationModel as Experimental_EvaluationModel,
|
|
4
|
+
EvaluationQuestion as Experimental_EvaluationQuestion,
|
|
5
|
+
EvaluationAnswer as Experimental_EvaluationAnswer,
|
|
6
|
+
EvaluationResult as Experimental_EvaluationResult,
|
|
7
|
+
} from './evaluation-result';
|