ai 7.0.101 → 7.0.103

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/CHANGELOG.md +49 -0
  2. package/dist/index.d.ts +138 -22
  3. package/dist/index.js +2373 -316
  4. package/dist/index.js.map +1 -1
  5. package/dist/internal/index.js +2 -1
  6. package/dist/internal/index.js.map +1 -1
  7. package/dist/test/index.d.ts +16 -2
  8. package/dist/test/index.js +17 -0
  9. package/dist/test/index.js.map +1 -1
  10. package/docs/03-agents/07-workflow-agent.mdx +1 -1
  11. package/docs/03-ai-sdk-core/18-code-mode.mdx +39 -0
  12. package/docs/03-ai-sdk-core/32-evaluation.mdx +110 -0
  13. package/docs/03-ai-sdk-core/36-realtime.mdx +41 -18
  14. package/docs/03-ai-sdk-core/42-batch.mdx +1 -2
  15. package/docs/04-ai-sdk-ui/03-chatbot-message-persistence.mdx +35 -0
  16. package/docs/04-ai-sdk-ui/21-transport.mdx +1 -1
  17. package/docs/06-advanced/11-secure-url-fetching.mdx +8 -2
  18. package/docs/07-reference/01-ai-sdk-core/14-evaluate.mdx +54 -0
  19. package/docs/07-reference/01-ai-sdk-core/32-validate-ui-messages.mdx +12 -0
  20. package/docs/07-reference/01-ai-sdk-core/33-safe-validate-ui-messages.mdx +12 -0
  21. package/docs/07-reference/02-ai-sdk-ui/05-use-realtime.mdx +364 -48
  22. package/docs/07-reference/02-ai-sdk-ui/31-convert-to-model-messages.mdx +12 -0
  23. package/docs/07-reference/04-ai-sdk-workflow/02-workflow-chat-transport.mdx +4 -4
  24. package/docs/07-reference/05-ai-sdk-errors/ai-evaluation-unsupported-question-type-error.mdx +31 -0
  25. package/docs/07-reference/05-ai-sdk-errors/index.mdx +131 -36
  26. package/package.json +12 -12
  27. package/src/error/index.ts +1 -0
  28. package/src/evaluate/evaluate.ts +112 -0
  29. package/src/evaluate/evaluation-result.ts +39 -0
  30. package/src/evaluate/index.ts +7 -0
  31. package/src/evaluate/validate-evaluation.ts +298 -0
  32. package/src/generate-text/execute-tools-from-stream.ts +7 -0
  33. package/src/generate-text/generate-text.ts +16 -12
  34. package/src/generate-text/stream-text.ts +9 -2
  35. package/src/generate-text/tool-caller-configuration.ts +58 -4
  36. package/src/index.ts +1 -0
  37. package/src/realtime/__fixtures__/fake-live-websocket.ts +71 -0
  38. package/src/realtime/__fixtures__/fake-realtime.ts +36 -0
  39. package/src/realtime/__fixtures__/fake-webrtc.ts +133 -0
  40. package/src/realtime/browser-realtime-audio.ts +107 -10
  41. package/src/realtime/browser-realtime-live-websocket.ts +247 -0
  42. package/src/realtime/browser-realtime-transport.ts +235 -69
  43. package/src/realtime/browser-realtime-webrtc.ts +582 -0
  44. package/src/realtime/encode-realtime-frame.ts +33 -0
  45. package/src/realtime/index.ts +1 -0
  46. package/src/realtime/realtime-attempt.ts +45 -0
  47. package/src/realtime/realtime-command-tracker.ts +81 -0
  48. package/src/realtime/realtime-event-channel.ts +170 -0
  49. package/src/realtime/realtime-event-reducer.ts +3 -0
  50. package/src/realtime/realtime-session-state.ts +65 -0
  51. package/src/realtime/realtime-session.ts +768 -218
  52. package/src/realtime/realtime-types.ts +1 -1
  53. package/src/realtime/validate-realtime-setup.ts +37 -0
  54. package/src/test/evaluation-mock-model-v4.ts +27 -0
  55. package/src/ui/convert-to-model-messages.ts +3 -0
  56. package/src/ui/process-ui-message-stream.ts +4 -0
  57. package/src/ui/ui-messages.ts +5 -1
  58. package/src/ui/validate-ui-messages.ts +3 -0
  59. package/src/ui/warn-if-ui-message-has-deprecated-raw-input.ts +36 -0
@@ -13,7 +13,7 @@ Unlike [`DefaultChatTransport`](/docs/ai-sdk-ui/transport) which assumes the ful
13
13
  'use client';
14
14
 
15
15
  import { useChat } from '@ai-sdk/react';
16
- import { WorkflowChatTransport } from '@ai-sdk/workflow';
16
+ import { WorkflowChatTransport } from '@ai-sdk/workflow/client';
17
17
 
18
18
  export default function Chat() {
19
19
  const { messages, sendMessage } = useChat({
@@ -30,7 +30,7 @@ export default function Chat() {
30
30
  ## Import
31
31
 
32
32
  <Snippet
33
- text={`import { WorkflowChatTransport } from "@ai-sdk/workflow"`}
33
+ text={`import { WorkflowChatTransport } from "@ai-sdk/workflow/client"`}
34
34
  prompt={false}
35
35
  />
36
36
 
@@ -238,7 +238,7 @@ See the [WorkflowAgent guide](/docs/agents/workflow-agent) for complete endpoint
238
238
  'use client';
239
239
 
240
240
  import { useChat } from '@ai-sdk/react';
241
- import { WorkflowChatTransport } from '@ai-sdk/workflow';
241
+ import { WorkflowChatTransport } from '@ai-sdk/workflow/client';
242
242
  import { useMemo } from 'react';
243
243
 
244
244
  export default function Chat() {
@@ -271,7 +271,7 @@ export default function Chat() {
271
271
  'use client';
272
272
 
273
273
  import { useChat } from '@ai-sdk/react';
274
- import { WorkflowChatTransport } from '@ai-sdk/workflow';
274
+ import { WorkflowChatTransport } from '@ai-sdk/workflow/client';
275
275
  import { useMemo } from 'react';
276
276
 
277
277
  export default function Chat() {
@@ -0,0 +1,31 @@
1
+ ---
2
+ title: AI_EvaluationUnsupportedQuestionTypeError
3
+ description: Identify an evaluation question type that a model does not support.
4
+ ---
5
+
6
+ # AI_EvaluationUnsupportedQuestionTypeError
7
+
8
+ This experimental error identifies a question whose type is unsupported by an
9
+ evaluation model. It is exported from `ai` and `@ai-sdk/provider` as
10
+ `Experimental_EvaluationUnsupportedQuestionTypeError`.
11
+
12
+ ## Properties
13
+
14
+ - `questionId`: The ID of the unsupported question.
15
+ - `questionType`: The requested question type.
16
+ - `provider`: The provider of the evaluation model.
17
+ - `modelId`: The evaluation model ID.
18
+ - `message`: A description of the unsupported question and model. Providers can
19
+ supply a custom message.
20
+
21
+ ## Checking for this Error
22
+
23
+ Use the marker-based `isInstance` check, which works across package copies:
24
+
25
+ ```typescript
26
+ import { Experimental_EvaluationUnsupportedQuestionTypeError as EvaluationUnsupportedQuestionTypeError } from 'ai';
27
+
28
+ if (EvaluationUnsupportedQuestionTypeError.isInstance(error)) {
29
+ console.log(error.questionId, error.questionType, error.modelId);
30
+ }
31
+ ```
@@ -1,43 +1,138 @@
1
1
  ---
2
2
  title: AI SDK Errors
3
- description: Troubleshooting information for common AI SDK errors.
3
+ description: Reference for AI SDK error classes and typed error handling.
4
4
  collapsed: true
5
5
  ---
6
6
 
7
7
  # AI SDK Errors
8
8
 
9
- - [AI_APICallError](/docs/reference/ai-sdk-errors/ai-api-call-error)
10
- - [AI_DownloadError](/docs/reference/ai-sdk-errors/ai-download-error)
11
- - [AI_EmptyResponseBodyError](/docs/reference/ai-sdk-errors/ai-empty-response-body-error)
12
- - [AI_InvalidArgumentError](/docs/reference/ai-sdk-errors/ai-invalid-argument-error)
13
- - [AI_InvalidDataContentError](/docs/reference/ai-sdk-errors/ai-invalid-data-content-error)
14
- - [AI_InvalidMessageRoleError](/docs/reference/ai-sdk-errors/ai-invalid-message-role-error)
15
- - [AI_InvalidPromptError](/docs/reference/ai-sdk-errors/ai-invalid-prompt-error)
16
- - [AI_InvalidResponseDataError](/docs/reference/ai-sdk-errors/ai-invalid-response-data-error)
17
- - [AI_InvalidToolApprovalError](/docs/reference/ai-sdk-errors/ai-invalid-tool-approval-error)
18
- - [AI_InvalidToolInputError](/docs/reference/ai-sdk-errors/ai-invalid-tool-input-error)
19
- - [AI_JSONParseError](/docs/reference/ai-sdk-errors/ai-json-parse-error)
20
- - [AI_LoadAPIKeyError](/docs/reference/ai-sdk-errors/ai-load-api-key-error)
21
- - [AI_LoadSettingError](/docs/reference/ai-sdk-errors/ai-load-setting-error)
22
- - [AI_MessageConversionError](/docs/reference/ai-sdk-errors/ai-message-conversion-error)
23
- - [AI_NoSpeechGeneratedError](/docs/reference/ai-sdk-errors/ai-no-speech-generated-error)
24
- - [AI_NoContentGeneratedError](/docs/reference/ai-sdk-errors/ai-no-content-generated-error)
25
- - [AI_NoImageGeneratedError](/docs/reference/ai-sdk-errors/ai-no-image-generated-error)
26
- - [AI_NoTranscriptGeneratedError](/docs/reference/ai-sdk-errors/ai-no-transcript-generated-error)
27
- - [AI_NoTranslationGeneratedError](/docs/reference/ai-sdk-errors/ai-no-translation-generated-error)
28
- - [AI_NoVideoGeneratedError](/docs/reference/ai-sdk-errors/ai-no-video-generated-error)
29
- - [AI_NoObjectGeneratedError](/docs/reference/ai-sdk-errors/ai-no-object-generated-error)
30
- - [AI_NoOutputGeneratedError](/docs/reference/ai-sdk-errors/ai-no-output-generated-error)
31
- - [AI_NoSuchModelError](/docs/reference/ai-sdk-errors/ai-no-such-model-error)
32
- - [AI_NoSuchProviderError](/docs/reference/ai-sdk-errors/ai-no-such-provider-error)
33
- - [AI_NoSuchProviderReferenceError](/docs/reference/ai-sdk-errors/ai-no-such-provider-reference-error)
34
- - [AI_NoSuchToolError](/docs/reference/ai-sdk-errors/ai-no-such-tool-error)
35
- - [AI_RetryError](/docs/reference/ai-sdk-errors/ai-retry-error)
36
- - [AI_StreamProviderError](/docs/reference/ai-sdk-errors/ai-stream-provider-error)
37
- - [AI_ToolCallNotFoundForApprovalError](/docs/reference/ai-sdk-errors/ai-tool-call-not-found-for-approval-error)
38
- - [AI_ToolCallRepairError](/docs/reference/ai-sdk-errors/ai-tool-call-repair-error)
39
- - [AI_ToolChoiceViolationError](/docs/reference/ai-sdk-errors/ai-tool-choice-violation-error)
40
- - [AI_TooManyEmbeddingValuesForCallError](/docs/reference/ai-sdk-errors/ai-too-many-embedding-values-for-call-error)
41
- - [AI_TypeValidationError](/docs/reference/ai-sdk-errors/ai-type-validation-error)
42
- - [AI_UIMessageStreamError](/docs/reference/ai-sdk-errors/ai-ui-message-stream-error)
43
- - [AI_UnsupportedFunctionalityError](/docs/reference/ai-sdk-errors/ai-unsupported-functionality-error)
9
+ The AI SDK exposes typed errors so applications can handle expected failure
10
+ modes without matching error message strings.
11
+
12
+ ## Importing Errors
13
+
14
+ The application package is named `ai`, not `@ai-sdk/ai`. It re-exports common
15
+ provider-level errors from `@ai-sdk/provider` together with the higher-level
16
+ errors raised by AI SDK Core:
17
+
18
+ ```typescript
19
+ import { APICallError, NoObjectGeneratedError } from 'ai';
20
+ ```
21
+
22
+ Provider implementations that depend directly on `@ai-sdk/provider` can import
23
+ its provider-level errors from that package instead:
24
+
25
+ ```typescript
26
+ import { APICallError, InvalidResponseDataError } from '@ai-sdk/provider';
27
+ ```
28
+
29
+ ## Migrating to Typed Error Handling
30
+
31
+ Replace message matching or an unconditionally generic `catch` block with the
32
+ most specific static `isInstance` guard available. Check `AISDKError` last when
33
+ you need a fallback for any AI SDK error:
34
+
35
+ ```typescript
36
+ import { AISDKError, APICallError, generateText } from 'ai';
37
+
38
+ try {
39
+ await generateText({
40
+ model,
41
+ prompt: 'Write a vegetarian lasagna recipe for 4 people.',
42
+ });
43
+ } catch (error) {
44
+ if (APICallError.isInstance(error)) {
45
+ console.error('Provider request failed:', error.statusCode);
46
+ return;
47
+ }
48
+
49
+ if (AISDKError.isInstance(error)) {
50
+ console.error('AI SDK error:', error.name);
51
+ return;
52
+ }
53
+
54
+ throw error;
55
+ }
56
+ ```
57
+
58
+ Prefer `ErrorClass.isInstance(error)` over `error instanceof ErrorClass` when
59
+ the class provides it. The static guard also works when multiple AI SDK package
60
+ versions are loaded.
61
+
62
+ ## Common Failure Modes
63
+
64
+ | Failure mode | Error to check |
65
+ | -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
66
+ | A provider request fails because of a network error or non-success response | [`APICallError`](/docs/reference/ai-sdk-errors/ai-api-call-error) |
67
+ | Automatic retries are exhausted | [`RetryError`](/docs/reference/ai-sdk-errors/ai-retry-error) |
68
+ | A provider reports an error after a response stream has started | [`StreamProviderError`](/docs/reference/ai-sdk-errors/ai-stream-provider-error) |
69
+ | A structured output cannot be parsed or validated | [`NoObjectGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-object-generated-error); inspect its `cause` for [`JSONParseError`](/docs/reference/ai-sdk-errors/ai-json-parse-error) or [`TypeValidationError`](/docs/reference/ai-sdk-errors/ai-type-validation-error) |
70
+ | A generation call returns no usable output | [`NoOutputGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-output-generated-error) or the modality-specific `No*GeneratedError` |
71
+ | A model calls a missing tool or supplies invalid tool input | [`NoSuchToolError`](/docs/reference/ai-sdk-errors/ai-no-such-tool-error) or [`InvalidToolInputError`](/docs/reference/ai-sdk-errors/ai-invalid-tool-input-error) |
72
+ | Tool-call repair fails | [`ToolCallRepairError`](/docs/reference/ai-sdk-errors/ai-tool-call-repair-error) |
73
+ | A response violates an enforced `toolChoice` | [`ToolChoiceViolationError`](/docs/reference/ai-sdk-errors/ai-tool-choice-violation-error) |
74
+ | Message history contains unresolved tool calls or invalid approvals | [`MissingToolResultsError`](/docs/troubleshooting/missing-tool-results-error), [`InvalidToolApprovalError`](/docs/reference/ai-sdk-errors/ai-invalid-tool-approval-error), or [`InvalidToolApprovalSignatureError`](/docs/reference/ai-sdk-errors/ai-invalid-tool-approval-signature-error) |
75
+ | A provider or model cannot be resolved | [`NoSuchProviderError`](/docs/reference/ai-sdk-errors/ai-no-such-provider-error), [`NoSuchModelError`](/docs/reference/ai-sdk-errors/ai-no-such-model-error), or [`NoSuchProviderReferenceError`](/docs/reference/ai-sdk-errors/ai-no-such-provider-reference-error) |
76
+ | A provider response is empty, malformed, or invalid | [`EmptyResponseBodyError`](/docs/reference/ai-sdk-errors/ai-empty-response-body-error), [`JSONParseError`](/docs/reference/ai-sdk-errors/ai-json-parse-error), or [`InvalidResponseDataError`](/docs/reference/ai-sdk-errors/ai-invalid-response-data-error) |
77
+ | The requested model or provider does not support a capability or specification version | [`UnsupportedFunctionalityError`](/docs/reference/ai-sdk-errors/ai-unsupported-functionality-error) or [`UnsupportedModelVersionError`](/docs/troubleshooting/unsupported-model-version) |
78
+ | UI messages cannot be converted or a UI message stream is invalid | [`MessageConversionError`](/docs/reference/ai-sdk-errors/ai-message-conversion-error) or [`UIMessageStreamError`](/docs/reference/ai-sdk-errors/ai-ui-message-stream-error) |
79
+
80
+ ## Error Reference
81
+
82
+ ### Base Error
83
+
84
+ - `AISDKError`: Base class and broad type guard for AI SDK errors.
85
+
86
+ ### Provider Requests and Responses
87
+
88
+ - [`APICallError`](/docs/reference/ai-sdk-errors/ai-api-call-error)
89
+ - [`DownloadError`](/docs/reference/ai-sdk-errors/ai-download-error)
90
+ - [`EmptyResponseBodyError`](/docs/reference/ai-sdk-errors/ai-empty-response-body-error)
91
+ - [`InvalidPromptError`](/docs/reference/ai-sdk-errors/ai-invalid-prompt-error)
92
+ - [`InvalidResponseDataError`](/docs/reference/ai-sdk-errors/ai-invalid-response-data-error)
93
+ - [`JSONParseError`](/docs/reference/ai-sdk-errors/ai-json-parse-error)
94
+ - [`LoadAPIKeyError`](/docs/reference/ai-sdk-errors/ai-load-api-key-error)
95
+ - [`LoadSettingError`](/docs/reference/ai-sdk-errors/ai-load-setting-error)
96
+ - [`NoContentGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-content-generated-error)
97
+ - [`NoSuchModelError`](/docs/reference/ai-sdk-errors/ai-no-such-model-error)
98
+ - [`NoSuchProviderReferenceError`](/docs/reference/ai-sdk-errors/ai-no-such-provider-reference-error)
99
+ - [`RetryError`](/docs/reference/ai-sdk-errors/ai-retry-error)
100
+ - [`StreamProviderError`](/docs/reference/ai-sdk-errors/ai-stream-provider-error)
101
+ - [`TooManyEmbeddingValuesForCallError`](/docs/reference/ai-sdk-errors/ai-too-many-embedding-values-for-call-error)
102
+ - [`TypeValidationError`](/docs/reference/ai-sdk-errors/ai-type-validation-error)
103
+ - [`UnsupportedFunctionalityError`](/docs/reference/ai-sdk-errors/ai-unsupported-functionality-error)
104
+
105
+ ### Inputs and Message Streams
106
+
107
+ - [`InvalidArgumentError`](/docs/reference/ai-sdk-errors/ai-invalid-argument-error)
108
+ - [`InvalidDataContentError`](/docs/reference/ai-sdk-errors/ai-invalid-data-content-error)
109
+ - [`InvalidMessageRoleError`](/docs/reference/ai-sdk-errors/ai-invalid-message-role-error)
110
+ - `InvalidStreamPartError`
111
+ - [`MessageConversionError`](/docs/reference/ai-sdk-errors/ai-message-conversion-error)
112
+ - [`UIMessageStreamError`](/docs/reference/ai-sdk-errors/ai-ui-message-stream-error)
113
+
114
+ ### Generated Output
115
+
116
+ - [`NoImageGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-image-generated-error)
117
+ - [`NoObjectGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-object-generated-error)
118
+ - [`NoOutputGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-output-generated-error)
119
+ - [`NoSpeechGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-speech-generated-error)
120
+ - [`NoTranscriptGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-transcript-generated-error)
121
+ - [`NoTranslationGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-translation-generated-error)
122
+ - [`NoVideoGeneratedError`](/docs/reference/ai-sdk-errors/ai-no-video-generated-error)
123
+
124
+ ### Providers and Model Compatibility
125
+
126
+ - [`NoSuchProviderError`](/docs/reference/ai-sdk-errors/ai-no-such-provider-error)
127
+ - [`UnsupportedModelVersionError`](/docs/troubleshooting/unsupported-model-version)
128
+
129
+ ### Tools and Approvals
130
+
131
+ - [`InvalidToolApprovalError`](/docs/reference/ai-sdk-errors/ai-invalid-tool-approval-error)
132
+ - [`InvalidToolApprovalSignatureError`](/docs/reference/ai-sdk-errors/ai-invalid-tool-approval-signature-error)
133
+ - [`InvalidToolInputError`](/docs/reference/ai-sdk-errors/ai-invalid-tool-input-error)
134
+ - [`MissingToolResultsError`](/docs/troubleshooting/missing-tool-results-error)
135
+ - [`NoSuchToolError`](/docs/reference/ai-sdk-errors/ai-no-such-tool-error)
136
+ - [`ToolCallNotFoundForApprovalError`](/docs/reference/ai-sdk-errors/ai-tool-call-not-found-for-approval-error)
137
+ - [`ToolCallRepairError`](/docs/reference/ai-sdk-errors/ai-tool-call-repair-error)
138
+ - [`ToolChoiceViolationError`](/docs/reference/ai-sdk-errors/ai-tool-choice-violation-error)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ai",
3
- "version": "7.0.101",
3
+ "version": "7.0.103",
4
4
  "type": "module",
5
5
  "description": "AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.",
6
6
  "license": "Apache-2.0",
@@ -42,20 +42,20 @@
42
42
  }
43
43
  },
44
44
  "dependencies": {
45
- "@ai-sdk/gateway": "4.0.81",
46
- "@ai-sdk/provider": "4.0.14",
47
- "@ai-sdk/provider-utils": "5.0.40"
45
+ "@ai-sdk/gateway": "4.0.83",
46
+ "@ai-sdk/provider": "4.0.16",
47
+ "@ai-sdk/provider-utils": "5.0.42"
48
48
  },
49
49
  "devDependencies": {
50
- "@ai-sdk/amazon-bedrock": "5.0.82",
51
- "@ai-sdk/deepseek": "3.0.44",
52
- "@ai-sdk/google": "4.0.70",
53
- "@ai-sdk/groq": "4.0.41",
54
- "@ai-sdk/huggingface": "2.0.48",
55
- "@ai-sdk/moonshotai": "3.0.49",
56
- "@ai-sdk/openai": "4.0.66",
50
+ "@ai-sdk/amazon-bedrock": "5.0.85",
51
+ "@ai-sdk/deepseek": "3.0.46",
52
+ "@ai-sdk/google": "4.0.73",
53
+ "@ai-sdk/groq": "4.0.43",
54
+ "@ai-sdk/huggingface": "2.0.50",
55
+ "@ai-sdk/moonshotai": "3.0.51",
56
+ "@ai-sdk/openai": "4.0.68",
57
57
  "@ai-sdk/test-server": "2.0.1",
58
- "@ai-sdk/xai": "4.0.58",
58
+ "@ai-sdk/xai": "5.0.1",
59
59
  "@edge-runtime/vm": "^5.0.0",
60
60
  "@smithy/eventstream-codec": "^4.3.3",
61
61
  "@smithy/util-utf8": "^4.3.3",
@@ -2,6 +2,7 @@ export {
2
2
  AISDKError,
3
3
  APICallError,
4
4
  EmptyResponseBodyError,
5
+ Experimental_EvaluationUnsupportedQuestionTypeError,
5
6
  InvalidPromptError,
6
7
  InvalidResponseDataError,
7
8
  JSONParseError,
@@ -0,0 +1,112 @@
1
+ import {
2
+ Experimental_EvaluationUnsupportedQuestionTypeError as EvaluationUnsupportedQuestionTypeError,
3
+ type Experimental_EvaluationModelV4CallOptions as EvaluationModelV4CallOptions,
4
+ } from '@ai-sdk/provider';
5
+ import {
6
+ withUserAgentSuffix,
7
+ type ProviderOptions,
8
+ } from '@ai-sdk/provider-utils';
9
+ import { UnsupportedModelVersionError } from '../error/unsupported-model-version-error';
10
+ import { logWarnings } from '../logger/log-warnings';
11
+ import { prepareRetries } from '../util/prepare-retries';
12
+ import { VERSION } from '../version';
13
+ import type {
14
+ EvaluationModel,
15
+ EvaluationQuestion,
16
+ EvaluationResult,
17
+ } from './evaluation-result';
18
+ import {
19
+ validateEvaluationInput,
20
+ validateEvaluationAnswers,
21
+ } from './validate-evaluation';
22
+
23
+ /** Evaluate typed questions against one shared state. Experimental. */
24
+ export async function evaluate<
25
+ const QUESTIONS extends Record<string, EvaluationQuestion>,
26
+ >({
27
+ model,
28
+ state,
29
+ questions,
30
+ maxRetries,
31
+ abortSignal,
32
+ headers,
33
+ providerOptions = {},
34
+ }: {
35
+ /** An evaluation model instance. String model resolution is not yet supported. */
36
+ model: EvaluationModel;
37
+ state: EvaluationModelV4CallOptions['state'];
38
+ questions: QUESTIONS;
39
+ /** Maximum retries for transient provider failures. Defaults to 2. */
40
+ maxRetries?: number;
41
+ abortSignal?: AbortSignal;
42
+ headers?: Record<string, string>;
43
+ providerOptions?: ProviderOptions;
44
+ }): Promise<EvaluationResult<QUESTIONS>> {
45
+ if (model.specificationVersion !== 'v4') {
46
+ throw new UnsupportedModelVersionError({
47
+ version: model.specificationVersion,
48
+ provider: model.provider,
49
+ modelId: model.modelId,
50
+ });
51
+ }
52
+
53
+ validateEvaluationInput({ state, questions });
54
+
55
+ for (const [questionId, question] of Object.entries(questions)) {
56
+ if (!model.supportedQuestionTypes.includes(question.type)) {
57
+ throw new EvaluationUnsupportedQuestionTypeError({
58
+ questionId,
59
+ questionType: question.type,
60
+ provider: model.provider,
61
+ modelId: model.modelId,
62
+ });
63
+ }
64
+ }
65
+
66
+ const { retry } = prepareRetries({ maxRetries, abortSignal });
67
+ const result = await retry(() => {
68
+ abortSignal?.throwIfAborted();
69
+ return model.doEvaluate({
70
+ state,
71
+ questions,
72
+ abortSignal,
73
+ headers: withUserAgentSuffix(headers ?? {}, `ai/${VERSION}`),
74
+ providerOptions,
75
+ });
76
+ });
77
+
78
+ abortSignal?.throwIfAborted();
79
+ validateEvaluationAnswers({
80
+ questions,
81
+ answers: result.answers,
82
+ rounding: result.rounding,
83
+ });
84
+ logWarnings({
85
+ warnings: result.warnings,
86
+ provider: model.provider,
87
+ model: model.modelId,
88
+ });
89
+
90
+ const inputTokens = result.usage?.inputTokens;
91
+ const outputTokens = result.usage?.outputTokens;
92
+
93
+ return {
94
+ answers: result.answers as EvaluationResult<QUESTIONS>['answers'],
95
+ usage: {
96
+ inputTokens,
97
+ outputTokens,
98
+ totalTokens:
99
+ inputTokens != null && outputTokens != null
100
+ ? inputTokens + outputTokens
101
+ : undefined,
102
+ },
103
+ warnings: result.warnings,
104
+ rounding: result.rounding,
105
+ providerMetadata: result.providerMetadata,
106
+ response: {
107
+ ...result.response,
108
+ timestamp: result.response?.timestamp ?? new Date(),
109
+ modelId: result.response?.modelId ?? model.modelId,
110
+ },
111
+ };
112
+ }
@@ -0,0 +1,39 @@
1
+ import type {
2
+ Experimental_EvaluationModelV4 as EvaluationModelV4,
3
+ Experimental_EvaluationModelV4Question as EvaluationModelV4Question,
4
+ Experimental_EvaluationModelV4Result as EvaluationModelV4Result,
5
+ } from '@ai-sdk/provider';
6
+
7
+ export type EvaluationModel = EvaluationModelV4;
8
+ export type EvaluationQuestion = EvaluationModelV4Question;
9
+
10
+ export type EvaluationAnswer<QUESTION extends EvaluationQuestion> =
11
+ QUESTION extends { type: 'choice'; criteria: infer CRITERIA }
12
+ ? {
13
+ type: 'choice';
14
+ choice: Extract<keyof CRITERIA, string>;
15
+ probabilities?: Record<Extract<keyof CRITERIA, string>, number>;
16
+ }
17
+ : QUESTION extends { type: 'score' }
18
+ ? { type: 'score'; score: number; probabilities?: Record<string, number> }
19
+ : { type: 'boolean'; probability: number };
20
+
21
+ export type EvaluationResult<
22
+ QUESTIONS extends Record<string, EvaluationQuestion>,
23
+ > = {
24
+ readonly answers: {
25
+ [ID in keyof QUESTIONS]: EvaluationAnswer<QUESTIONS[ID]>;
26
+ };
27
+ readonly usage: {
28
+ inputTokens: number | undefined;
29
+ outputTokens: number | undefined;
30
+ totalTokens: number | undefined;
31
+ };
32
+ readonly warnings: EvaluationModelV4Result['warnings'];
33
+ readonly rounding: EvaluationModelV4Result['rounding'];
34
+ readonly providerMetadata: EvaluationModelV4Result['providerMetadata'];
35
+ readonly response: NonNullable<EvaluationModelV4Result['response']> & {
36
+ timestamp: Date;
37
+ modelId: string;
38
+ };
39
+ };
@@ -0,0 +1,7 @@
1
+ export { evaluate as experimental_evaluate } from './evaluate';
2
+ export type {
3
+ EvaluationModel as Experimental_EvaluationModel,
4
+ EvaluationQuestion as Experimental_EvaluationQuestion,
5
+ EvaluationAnswer as Experimental_EvaluationAnswer,
6
+ EvaluationResult as Experimental_EvaluationResult,
7
+ } from './evaluation-result';