@convex-dev/agent 0.5.0-alpha.1 → 0.6.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/README.md +32 -27
  2. package/dist/UIMessages.d.ts +46 -0
  3. package/dist/UIMessages.d.ts.map +1 -0
  4. package/dist/UIMessages.js +546 -0
  5. package/dist/UIMessages.js.map +1 -0
  6. package/dist/client/createTool.d.ts +129 -27
  7. package/dist/client/createTool.d.ts.map +1 -1
  8. package/dist/client/createTool.js +66 -12
  9. package/dist/client/createTool.js.map +1 -1
  10. package/dist/client/defaultComponent.d.ts +11 -0
  11. package/dist/client/defaultComponent.d.ts.map +1 -0
  12. package/dist/client/defaultComponent.js +7 -0
  13. package/dist/client/defaultComponent.js.map +1 -0
  14. package/dist/client/definePlaygroundAPI.d.ts +1323 -192
  15. package/dist/client/definePlaygroundAPI.d.ts.map +1 -1
  16. package/dist/client/definePlaygroundAPI.js +52 -28
  17. package/dist/client/definePlaygroundAPI.js.map +1 -1
  18. package/dist/client/files.d.ts +20 -7
  19. package/dist/client/files.d.ts.map +1 -1
  20. package/dist/client/files.js +68 -11
  21. package/dist/client/files.js.map +1 -1
  22. package/dist/client/index.d.ts +1056 -965
  23. package/dist/client/index.d.ts.map +1 -1
  24. package/dist/client/index.js +242 -748
  25. package/dist/client/index.js.map +1 -1
  26. package/dist/client/messages.d.ts +461 -0
  27. package/dist/client/messages.d.ts.map +1 -0
  28. package/dist/client/messages.js +106 -0
  29. package/dist/client/messages.js.map +1 -0
  30. package/dist/client/mockModel.d.ts +42 -0
  31. package/dist/client/mockModel.d.ts.map +1 -0
  32. package/dist/client/mockModel.js +175 -0
  33. package/dist/client/mockModel.js.map +1 -0
  34. package/dist/client/saveInputMessages.d.ts +20 -0
  35. package/dist/client/saveInputMessages.d.ts.map +1 -0
  36. package/dist/client/saveInputMessages.js +58 -0
  37. package/dist/client/saveInputMessages.js.map +1 -0
  38. package/dist/client/search.d.ts +346 -35
  39. package/dist/client/search.d.ts.map +1 -1
  40. package/dist/client/search.js +350 -39
  41. package/dist/client/search.js.map +1 -1
  42. package/dist/client/start.d.ts +84 -0
  43. package/dist/client/start.d.ts.map +1 -0
  44. package/dist/client/start.js +171 -0
  45. package/dist/client/start.js.map +1 -0
  46. package/dist/client/streamText.d.ts +46 -0
  47. package/dist/client/streamText.d.ts.map +1 -0
  48. package/dist/client/streamText.js +93 -0
  49. package/dist/client/streamText.js.map +1 -0
  50. package/dist/client/streaming.d.ts +3705 -32
  51. package/dist/client/streaming.d.ts.map +1 -1
  52. package/dist/client/streaming.js +141 -59
  53. package/dist/client/streaming.js.map +1 -1
  54. package/dist/client/threads.d.ts +46 -0
  55. package/dist/client/threads.d.ts.map +1 -0
  56. package/dist/client/threads.js +49 -0
  57. package/dist/client/threads.js.map +1 -0
  58. package/dist/client/types.d.ts +265 -128
  59. package/dist/client/types.d.ts.map +1 -1
  60. package/dist/client/utils.d.ts +4 -0
  61. package/dist/client/utils.d.ts.map +1 -0
  62. package/dist/client/utils.js +21 -0
  63. package/dist/client/utils.js.map +1 -0
  64. package/dist/component/_generated/api.d.ts +24 -2178
  65. package/dist/component/_generated/api.d.ts.map +1 -1
  66. package/dist/component/_generated/api.js +10 -1
  67. package/dist/component/_generated/api.js.map +1 -1
  68. package/dist/component/_generated/component.d.ts +3119 -0
  69. package/dist/component/_generated/component.d.ts.map +1 -0
  70. package/dist/component/_generated/component.js +11 -0
  71. package/dist/component/_generated/component.js.map +1 -0
  72. package/dist/component/_generated/dataModel.d.ts +4 -18
  73. package/dist/component/_generated/dataModel.d.ts.map +1 -0
  74. package/dist/component/_generated/dataModel.js +11 -0
  75. package/dist/component/_generated/dataModel.js.map +1 -0
  76. package/dist/component/_generated/server.d.ts +10 -38
  77. package/dist/component/_generated/server.d.ts.map +1 -1
  78. package/dist/component/_generated/server.js +9 -5
  79. package/dist/component/_generated/server.js.map +1 -1
  80. package/dist/component/files.d.ts +16 -10
  81. package/dist/component/files.d.ts.map +1 -1
  82. package/dist/component/files.js +10 -2
  83. package/dist/component/files.js.map +1 -1
  84. package/dist/component/messages.d.ts +2553 -342
  85. package/dist/component/messages.d.ts.map +1 -1
  86. package/dist/component/messages.js +387 -154
  87. package/dist/component/messages.js.map +1 -1
  88. package/dist/component/schema.d.ts +5697 -3584
  89. package/dist/component/schema.d.ts.map +1 -1
  90. package/dist/component/schema.js +18 -41
  91. package/dist/component/schema.js.map +1 -1
  92. package/dist/component/streams.d.ts +35 -335
  93. package/dist/component/streams.d.ts.map +1 -1
  94. package/dist/component/streams.js +114 -73
  95. package/dist/component/streams.js.map +1 -1
  96. package/dist/component/threads.d.ts +16 -16
  97. package/dist/component/users.d.ts +4 -4
  98. package/dist/component/vector/index.d.ts +1 -1
  99. package/dist/component/vector/index.d.ts.map +1 -1
  100. package/dist/component/vector/index.js +1 -3
  101. package/dist/component/vector/index.js.map +1 -1
  102. package/dist/deltas.d.ts +43 -0
  103. package/dist/deltas.d.ts.map +1 -0
  104. package/dist/deltas.js +447 -0
  105. package/dist/deltas.js.map +1 -0
  106. package/dist/mapping.d.ts +20 -20
  107. package/dist/mapping.d.ts.map +1 -1
  108. package/dist/mapping.js +313 -96
  109. package/dist/mapping.js.map +1 -1
  110. package/dist/react/SmoothText.d.ts +5 -0
  111. package/dist/react/SmoothText.d.ts.map +1 -0
  112. package/dist/react/SmoothText.js +6 -0
  113. package/dist/react/SmoothText.js.map +1 -0
  114. package/dist/react/index.d.ts +5 -77
  115. package/dist/react/index.d.ts.map +1 -1
  116. package/dist/react/index.js +6 -160
  117. package/dist/react/index.js.map +1 -1
  118. package/dist/react/optimisticallySendMessage.d.ts +36 -3
  119. package/dist/react/optimisticallySendMessage.d.ts.map +1 -1
  120. package/dist/react/optimisticallySendMessage.js +35 -9
  121. package/dist/react/optimisticallySendMessage.js.map +1 -1
  122. package/dist/react/types.d.ts +4 -18
  123. package/dist/react/types.d.ts.map +1 -1
  124. package/dist/react/useDeltaStreams.d.ts +10 -0
  125. package/dist/react/useDeltaStreams.d.ts.map +1 -0
  126. package/dist/react/useDeltaStreams.js +101 -0
  127. package/dist/react/useDeltaStreams.js.map +1 -0
  128. package/dist/react/useSmoothText.d.ts +13 -12
  129. package/dist/react/useSmoothText.d.ts.map +1 -1
  130. package/dist/react/useSmoothText.js +32 -15
  131. package/dist/react/useSmoothText.js.map +1 -1
  132. package/dist/react/useStreamingUIMessages.d.ts +22 -0
  133. package/dist/react/useStreamingUIMessages.d.ts.map +1 -0
  134. package/dist/react/useStreamingUIMessages.js +92 -0
  135. package/dist/react/useStreamingUIMessages.js.map +1 -0
  136. package/dist/react/useThreadMessages.d.ts +104 -0
  137. package/dist/react/useThreadMessages.d.ts.map +1 -0
  138. package/dist/react/useThreadMessages.js +148 -0
  139. package/dist/react/useThreadMessages.js.map +1 -0
  140. package/dist/react/useUIMessages.d.ts +96 -0
  141. package/dist/react/useUIMessages.d.ts.map +1 -0
  142. package/dist/react/useUIMessages.js +108 -0
  143. package/dist/react/useUIMessages.js.map +1 -0
  144. package/dist/shared.d.ts +20 -4
  145. package/dist/shared.d.ts.map +1 -1
  146. package/dist/shared.js +45 -8
  147. package/dist/shared.js.map +1 -1
  148. package/dist/validators.d.ts +22981 -5666
  149. package/dist/validators.d.ts.map +1 -1
  150. package/dist/validators.js +245 -137
  151. package/dist/validators.js.map +1 -1
  152. package/package.json +98 -50
  153. package/src/UIMessages.combineUIMessages.test.ts +239 -0
  154. package/src/UIMessages.test.ts +273 -0
  155. package/src/UIMessages.ts +739 -0
  156. package/src/client/createTool.ts +293 -76
  157. package/src/client/defaultComponent.ts +17 -0
  158. package/src/client/definePlaygroundAPI.ts +67 -31
  159. package/src/client/files.ts +100 -20
  160. package/src/client/index.test.ts +40 -85
  161. package/src/client/index.ts +520 -1290
  162. package/src/client/messages.ts +237 -0
  163. package/src/client/mockModel.ts +245 -0
  164. package/src/client/saveInputMessages.test.ts +583 -0
  165. package/src/client/saveInputMessages.ts +101 -0
  166. package/src/client/search.test.ts +1207 -0
  167. package/src/client/search.ts +577 -70
  168. package/src/client/start.ts +310 -0
  169. package/src/client/streamText.ts +163 -0
  170. package/src/client/streaming.test.ts +186 -0
  171. package/src/client/streaming.ts +219 -97
  172. package/src/client/threads.ts +83 -0
  173. package/src/client/types.ts +368 -219
  174. package/src/client/utils.ts +27 -0
  175. package/src/component/_generated/api.ts +64 -0
  176. package/src/component/_generated/component.ts +4913 -0
  177. package/src/component/_generated/{server.d.ts → server.ts} +33 -21
  178. package/src/component/files.ts +11 -2
  179. package/src/component/messages.test.ts +195 -51
  180. package/src/component/messages.ts +490 -201
  181. package/src/component/schema.ts +20 -46
  182. package/src/component/setup.test.ts +7 -0
  183. package/src/component/streams.ts +184 -83
  184. package/src/component/users.test.ts +0 -1
  185. package/src/component/vector/index.ts +1 -3
  186. package/src/deltas.test.ts +626 -0
  187. package/src/deltas.ts +570 -0
  188. package/src/fromUIMessages.test.ts +497 -0
  189. package/src/mapping.test.ts +103 -6
  190. package/src/mapping.ts +422 -161
  191. package/src/react/SmoothText.tsx +9 -0
  192. package/src/react/index.ts +10 -230
  193. package/src/react/optimisticallySendMessage.ts +55 -12
  194. package/src/react/types.ts +6 -39
  195. package/src/react/useDeltaStreams.ts +154 -0
  196. package/src/react/useSmoothText.ts +56 -36
  197. package/src/react/useStreamingUIMessages.ts +143 -0
  198. package/src/react/useThreadMessages.ts +262 -0
  199. package/src/react/useUIMessages.test.ts +255 -0
  200. package/src/react/useUIMessages.ts +195 -0
  201. package/src/shared.ts +88 -12
  202. package/src/test.ts +18 -0
  203. package/src/toUIMessages.test.ts +1269 -0
  204. package/src/validators.test.ts +18 -19
  205. package/src/validators.ts +325 -185
  206. package/dist/client/_generated/_ignore.d.ts +0 -1
  207. package/dist/client/_generated/_ignore.d.ts.map +0 -1
  208. package/dist/client/_generated/_ignore.js +0 -3
  209. package/dist/client/_generated/_ignore.js.map +0 -1
  210. package/dist/client/listMessages.d.ts +0 -22
  211. package/dist/client/listMessages.d.ts.map +0 -1
  212. package/dist/client/listMessages.js +0 -25
  213. package/dist/client/listMessages.js.map +0 -1
  214. package/dist/package.json +0 -3
  215. package/dist/react/deltas.d.ts +0 -26
  216. package/dist/react/deltas.d.ts.map +0 -1
  217. package/dist/react/deltas.js +0 -384
  218. package/dist/react/deltas.js.map +0 -1
  219. package/dist/react/toUIMessages.d.ts +0 -15
  220. package/dist/react/toUIMessages.d.ts.map +0 -1
  221. package/dist/react/toUIMessages.js +0 -211
  222. package/dist/react/toUIMessages.js.map +0 -1
  223. package/src/client/listMessages.ts +0 -38
  224. package/src/component/_generated/api.d.ts +0 -2202
  225. package/src/component/_generated/api.js +0 -23
  226. package/src/component/_generated/server.js +0 -90
  227. package/src/node_modules/.vite/vitest/da39a3ee5e6b4b0d3255bfef95601890afd80709/results.json +0 -1
  228. package/src/react/deltas.test.ts +0 -315
  229. package/src/react/deltas.ts +0 -478
  230. package/src/react/toUIMessages.test.ts +0 -420
  231. package/src/react/toUIMessages.ts +0 -253
  232. package/src/vitest.config.ts +0 -7
  233. /package/src/component/_generated/{dataModel.d.ts → dataModel.ts} +0 -0
package/src/mapping.ts CHANGED
@@ -16,27 +16,43 @@ import {
16
16
  type TextPart,
17
17
  type ToolCallPart,
18
18
  type ToolResultPart,
19
- type ReasoningUIPart,
19
+ type ProviderMetadata,
20
+ type JSONValue,
20
21
  } from "ai";
21
- import type {
22
- Message,
23
- MessageWithMetadata,
24
- Usage,
25
- vContent,
26
- vFilePart,
27
- vImagePart,
28
- vReasoningPart,
29
- vRedactedReasoningPart,
30
- vTextPart,
31
- vToolCallPart,
32
- vToolResultPart,
22
+ import {
23
+ vMessageWithMetadata,
24
+ type vSourcePart,
25
+ type Message,
26
+ type MessageWithMetadata,
27
+ type Usage,
28
+ type vFilePart,
29
+ type vImagePart,
30
+ type vReasoningPart,
31
+ type vRedactedReasoningPart,
32
+ type vTextPart,
33
+ type vToolCallPart,
34
+ type vToolResultPart,
35
+ type SourcePart,
36
+ vToolResultOutput,
37
+ type MessageDoc,
38
+ vToolApprovalRequest,
39
+ vToolApprovalResponse,
33
40
  } from "./validators.js";
34
41
  import type { ActionCtx, AgentComponent } from "./client/types.js";
35
- import type { RunMutationCtx } from "./client/types.js";
42
+ import type { MutationCtx } from "./client/types.js";
36
43
  import { MAX_FILE_SIZE, storeFile } from "./client/files.js";
37
44
  import type { Infer } from "convex/values";
38
- import { omit } from "convex-helpers";
39
-
45
+ import {
46
+ convertUint8ArrayToBase64,
47
+ type ProviderOptions,
48
+ type ReasoningPart,
49
+ } from "@ai-sdk/provider-utils";
50
+ import { parse, validate } from "convex-helpers/validators";
51
+ import {
52
+ getModelName,
53
+ getProviderName,
54
+ type ModelOrMetadata,
55
+ } from "./shared.js";
40
56
  export type AIMessageWithoutId = Omit<AIMessage, "id">;
41
57
 
42
58
  export type SerializeUrlsAndUint8Arrays<T> = T extends URL
@@ -45,8 +61,7 @@ export type SerializeUrlsAndUint8Arrays<T> = T extends URL
45
61
  ? ArrayBuffer
46
62
  : T extends Array<infer Inner>
47
63
  ? Array<SerializeUrlsAndUint8Arrays<Inner>>
48
- : // eslint-disable-next-line @typescript-eslint/no-explicit-any
49
- T extends Record<string, any>
64
+ : T extends Record<string, any>
50
65
  ? { [K in keyof T]: SerializeUrlsAndUint8Arrays<T[K]> }
51
66
  : T;
52
67
 
@@ -56,12 +71,10 @@ export type SerializedContent = Message["content"];
56
71
  export type SerializedMessage = Message;
57
72
 
58
73
  export async function serializeMessage(
59
- ctx: ActionCtx | RunMutationCtx,
74
+ ctx: ActionCtx | MutationCtx,
60
75
  component: AgentComponent,
61
- messageWithId: (ModelMessage & { id?: string }) | Message,
76
+ message: ModelMessage | Message,
62
77
  ): Promise<{ message: SerializedMessage; fileIds?: string[] }> {
63
- const message =
64
- "id" in messageWithId ? omit(messageWithId, ["id"]) : messageWithId;
65
78
  const { content, fileIds } = await serializeContent(
66
79
  ctx,
67
80
  component,
@@ -69,36 +82,91 @@ export async function serializeMessage(
69
82
  );
70
83
  return {
71
84
  message: {
72
- ...message,
85
+ role: message.role,
73
86
  content,
87
+ ...(message.providerOptions
88
+ ? { providerOptions: message.providerOptions }
89
+ : {}),
74
90
  } as SerializedMessage,
75
91
  fileIds,
76
92
  };
77
93
  }
78
94
 
79
- export function deserializeMessage(message: SerializedMessage): ModelMessage {
95
+ // Similar to serializeMessage, but doesn't save any files and is looser
96
+ // For use on the frontend / in synchronous environments.
97
+ export function fromModelMessage(message: ModelMessage): Message {
98
+ const content = fromModelMessageContent(message.content);
99
+ return {
100
+ role: message.role,
101
+ content,
102
+ ...(message.providerOptions
103
+ ? { providerOptions: message.providerOptions }
104
+ : {}),
105
+ } as SerializedMessage;
106
+ }
107
+
108
+ export async function serializeOrThrow(
109
+ message: ModelMessage | Message,
110
+ ): Promise<SerializedMessage> {
111
+ const { content } = await serializeContent(
112
+ {} as any,
113
+ {} as any,
114
+ message.content,
115
+ );
116
+ return {
117
+ role: message.role,
118
+ content,
119
+ ...(message.providerOptions
120
+ ? { providerOptions: message.providerOptions }
121
+ : {}),
122
+ } as SerializedMessage;
123
+ }
124
+
125
+ export function toModelMessage(
126
+ message: SerializedMessage | ModelMessage,
127
+ ): ModelMessage {
80
128
  return {
81
129
  ...message,
82
- content: deserializeContent(message.content),
130
+ content: toModelMessageContent(message.content),
83
131
  } as ModelMessage;
84
132
  }
85
133
 
134
+ export function docsToModelMessages(messages: MessageDoc[]): ModelMessage[] {
135
+ return messages
136
+ .map((m) => m.message)
137
+ .filter((m) => !!m)
138
+ .filter((m) => !!m.content.length)
139
+ .map(toModelMessage);
140
+ }
141
+
86
142
  export function serializeUsage(usage: LanguageModelUsage): Usage {
87
143
  return {
88
144
  promptTokens: usage.inputTokens ?? 0,
89
145
  completionTokens: usage.outputTokens ?? 0,
90
146
  totalTokens: usage.totalTokens ?? 0,
91
147
  reasoningTokens: usage.reasoningTokens,
148
+ cachedInputTokens: usage.cachedInputTokens,
92
149
  };
93
150
  }
94
151
 
95
- export function deserializeUsage(usage: Usage): LanguageModelUsage {
152
+ export function toModelMessageUsage(usage: Usage): LanguageModelUsage {
96
153
  return {
97
154
  inputTokens: usage.promptTokens,
98
155
  outputTokens: usage.completionTokens,
99
156
  totalTokens: usage.totalTokens,
100
157
  reasoningTokens: usage.reasoningTokens,
101
158
  cachedInputTokens: usage.cachedInputTokens,
159
+ // These detail fields are required by LanguageModelUsage type but we don't
160
+ // have the granular data, so we provide empty objects with undefined values.
161
+ inputTokenDetails: {
162
+ cacheReadTokens: undefined,
163
+ cacheWriteTokens: undefined,
164
+ noCacheTokens: undefined,
165
+ },
166
+ outputTokenDetails: {
167
+ textTokens: undefined,
168
+ reasoningTokens: undefined,
169
+ },
102
170
  };
103
171
  }
104
172
 
@@ -109,34 +177,47 @@ export function serializeWarnings(
109
177
  return undefined;
110
178
  }
111
179
  return warnings.map((warning) => {
112
- if (warning.type !== "unsupported-setting") {
113
- return warning;
180
+ if (warning.type === "compatibility") {
181
+ return {
182
+ type: "unsupported-setting",
183
+ setting: warning.feature,
184
+ details: warning.details,
185
+ };
114
186
  }
115
- return {
116
- ...warning,
117
- setting: warning.setting.toString(),
118
- };
119
- });
187
+ return warning;
188
+ }) as any;
120
189
  }
121
190
 
122
- export function deserializeWarnings(
191
+ export function toModelMessageWarnings(
123
192
  warnings: MessageWithMetadata["warnings"],
124
193
  ): CallWarning[] | undefined {
125
- // We don't need to do anythign here for now
126
- return warnings;
194
+ if (!warnings) {
195
+ return undefined;
196
+ }
197
+ return warnings.map((warning) => {
198
+ if (warning.type === "unsupported-setting") {
199
+ return {
200
+ type: "compatibility",
201
+ feature: warning.setting,
202
+ details: warning.details,
203
+ };
204
+ }
205
+ return warning;
206
+ }) as any;
127
207
  }
128
208
 
129
209
  export async function serializeNewMessagesInStep<TOOLS extends ToolSet>(
130
210
  ctx: ActionCtx,
131
211
  component: AgentComponent,
132
212
  step: StepResult<TOOLS>,
133
- metadata: { model: string; provider: string },
134
- ): Promise<MessageWithMetadata[]> {
213
+ model: ModelOrMetadata | undefined,
214
+ ): Promise<{ messages: MessageWithMetadata[] }> {
135
215
  // If there are tool results, there's another message with the tool results
136
- // ref: https://github.com/vercel/ai/blob/main/packages/ai/core/generate-text/to-response-messages.ts
216
+ // ref: https://github.com/vercel/ai/blob/main/packages/ai/src/generate-text/to-response-messages.ts#L120
217
+ const hasToolMessage = step.response.messages.at(-1)?.role === "tool";
137
218
  const assistantFields = {
138
- model: metadata.model,
139
- provider: metadata.provider,
219
+ model: model ? getModelName(model) : undefined,
220
+ provider: model ? getProviderName(model) : undefined,
140
221
  providerMetadata: step.providerMetadata,
141
222
  reasoning: step.reasoningText,
142
223
  reasoningDetails: step.reasoning,
@@ -144,40 +225,34 @@ export async function serializeNewMessagesInStep<TOOLS extends ToolSet>(
144
225
  warnings: serializeWarnings(step.warnings),
145
226
  finishReason: step.finishReason,
146
227
  // Only store the sources on one message
147
- sources: step.toolResults.length === 0 ? step.sources : undefined,
228
+ sources: hasToolMessage ? undefined : step.sources,
148
229
  } satisfies Omit<MessageWithMetadata, "message" | "text" | "fileIds">;
149
- const toolFields = {
150
- sources: step.sources,
151
- };
230
+ const toolFields = { sources: step.sources };
152
231
  const messages: MessageWithMetadata[] = await Promise.all(
153
- (step.toolResults.length > 0
232
+ (hasToolMessage
154
233
  ? step.response.messages.slice(-2)
155
- : step.response.messages.slice(-1)
156
- ).map(async (messageWithId): Promise<MessageWithMetadata> => {
157
- const { message, fileIds } = await serializeMessage(
158
- ctx,
159
- component,
160
- messageWithId,
161
- );
162
- return {
234
+ : step.content.length
235
+ ? step.response.messages.slice(-1)
236
+ : [{ role: "assistant" as const, content: [] }]
237
+ ).map(async (msg): Promise<MessageWithMetadata> => {
238
+ const { message, fileIds } = await serializeMessage(ctx, component, msg);
239
+ return parse(vMessageWithMetadata, {
163
240
  message,
164
- // Let's not store the ID by default here. It's being generated internally
165
- // and not referenced elsewhere that we know of.
166
- // id: message.id,
167
241
  ...(message.role === "tool" ? toolFields : assistantFields),
168
242
  text: step.text,
169
243
  fileIds,
170
- };
244
+ });
171
245
  }),
172
246
  );
173
- return messages;
247
+ // TODO: capture step.files separately?
248
+ return { messages };
174
249
  }
175
250
 
176
251
  export async function serializeObjectResult(
177
252
  ctx: ActionCtx,
178
253
  component: AgentComponent,
179
254
  result: GenerateObjectResult<unknown>,
180
- metadata: { model: string; provider: string },
255
+ model: ModelOrMetadata | undefined,
181
256
  ): Promise<{ messages: MessageWithMetadata[] }> {
182
257
  const text = JSON.stringify(result.object);
183
258
 
@@ -189,9 +264,8 @@ export async function serializeObjectResult(
189
264
  messages: [
190
265
  {
191
266
  message,
192
- id: result.response.id,
193
- model: metadata.model,
194
- provider: metadata.provider,
267
+ model: model ? getModelName(model) : undefined,
268
+ provider: model ? getProviderName(model) : undefined,
195
269
  providerMetadata: result.providerMetadata,
196
270
  finishReason: result.finishReason,
197
271
  text,
@@ -203,8 +277,18 @@ export async function serializeObjectResult(
203
277
  };
204
278
  }
205
279
 
280
+ function getMimeOrMediaType(part: { mediaType?: string; mimeType?: string }) {
281
+ if ("mediaType" in part) {
282
+ return part.mediaType;
283
+ }
284
+ if ("mimeType" in part) {
285
+ return part.mimeType;
286
+ }
287
+ return undefined;
288
+ }
289
+
206
290
  export async function serializeContent(
207
- ctx: ActionCtx | RunMutationCtx,
291
+ ctx: ActionCtx | MutationCtx,
208
292
  component: AgentComponent,
209
293
  content: Content | Message["content"],
210
294
  ): Promise<{ content: SerializedContent; fileIds?: string[] }> {
@@ -212,23 +296,24 @@ export async function serializeContent(
212
296
  return { content };
213
297
  }
214
298
  const fileIds: string[] = [];
215
- function getMimeType(part: { mediaType?: string; mimeType?: string }) {
216
- if ("mediaType" in part) {
217
- return part.mediaType;
218
- }
219
- if ("mimeType" in part) {
220
- return part.mimeType;
221
- }
222
- return undefined;
223
- }
224
299
  const serialized = await Promise.all(
225
300
  content.map(async (part) => {
301
+ const metadata: {
302
+ providerOptions?: ProviderOptions;
303
+ providerMetadata?: ProviderMetadata;
304
+ } = {};
305
+ if ("providerOptions" in part) {
306
+ metadata.providerOptions = part.providerOptions as ProviderOptions;
307
+ }
308
+ if ("providerMetadata" in part) {
309
+ metadata.providerMetadata = part.providerMetadata as ProviderMetadata;
310
+ }
226
311
  switch (part.type) {
227
312
  case "text": {
228
313
  return {
229
314
  type: part.type,
230
315
  text: part.text,
231
- providerOptions: part.providerOptions,
316
+ ...metadata,
232
317
  } satisfies Infer<typeof vTextPart>;
233
318
  }
234
319
  case "image": {
@@ -241,7 +326,7 @@ export async function serializeContent(
241
326
  ctx,
242
327
  component,
243
328
  new Blob([image], {
244
- type: getMimeType(part) || guessMimeType(image),
329
+ type: getMimeOrMediaType(part) || guessMimeType(image),
245
330
  }),
246
331
  );
247
332
  image = file.url;
@@ -249,8 +334,8 @@ export async function serializeContent(
249
334
  }
250
335
  return {
251
336
  type: part.type,
252
- mimeType: getMimeType(part),
253
- providerOptions: part.providerOptions,
337
+ mediaType: getMimeOrMediaType(part),
338
+ ...metadata,
254
339
  image,
255
340
  } satisfies Infer<typeof vImagePart>;
256
341
  }
@@ -260,7 +345,7 @@ export async function serializeContent(
260
345
  const { file } = await storeFile(
261
346
  ctx,
262
347
  component,
263
- new Blob([data], { type: getMimeType(part) }),
348
+ new Blob([data], { type: getMimeOrMediaType(part) }),
264
349
  );
265
350
  data = file.url;
266
351
  fileIds.push(file.fileId);
@@ -269,118 +354,295 @@ export async function serializeContent(
269
354
  type: part.type,
270
355
  data,
271
356
  filename: part.filename,
272
- mimeType: getMimeType(part)!,
273
- providerOptions: part.providerOptions,
357
+ mediaType: getMimeOrMediaType(part)!,
358
+ ...metadata,
274
359
  } satisfies Infer<typeof vFilePart>;
275
360
  }
276
361
  case "tool-call": {
277
- const args = "input" in part ? part.input : part.args;
362
+ // Handle legacy data where only args field exists
363
+ const input = part.input ?? (part as any)?.args ?? {};
278
364
  return {
279
365
  type: part.type,
280
- args: args ?? null,
366
+ input,
367
+ /** @deprecated Use `input` instead. */
368
+ args: input,
281
369
  toolCallId: part.toolCallId,
282
370
  toolName: part.toolName,
283
- providerOptions: part.providerOptions,
284
371
  providerExecuted: part.providerExecuted,
372
+ ...metadata,
285
373
  } satisfies Infer<typeof vToolCallPart>;
286
374
  }
287
375
  case "tool-result": {
288
- const result = "output" in part ? part.output : part.result;
289
- return {
290
- type: part.type,
291
- result: result ?? null,
292
- toolCallId: part.toolCallId,
293
- toolName: part.toolName,
294
- providerOptions: part.providerOptions,
295
- } satisfies Infer<typeof vToolResultPart>;
376
+ return normalizeToolResult(part, metadata);
296
377
  }
297
378
  case "reasoning": {
298
379
  return {
299
380
  type: part.type,
300
381
  text: part.text,
301
- providerOptions: part.providerOptions,
382
+ ...metadata,
302
383
  } satisfies Infer<typeof vReasoningPart>;
303
384
  }
385
+ // Not in current generation output, but could be in historical messages
304
386
  case "redacted-reasoning": {
305
387
  return {
306
388
  type: part.type,
307
389
  data: part.data,
308
- providerOptions: part.providerOptions,
390
+ ...metadata,
309
391
  } satisfies Infer<typeof vRedactedReasoningPart>;
310
392
  }
393
+ case "source": {
394
+ return part satisfies Infer<typeof vSourcePart>;
395
+ }
396
+ case "tool-approval-request": {
397
+ return {
398
+ type: part.type,
399
+ approvalId: part.approvalId,
400
+ toolCallId: part.toolCallId,
401
+ ...metadata,
402
+ } satisfies Infer<typeof vToolApprovalRequest>;
403
+ }
404
+ case "tool-approval-response": {
405
+ return {
406
+ type: part.type,
407
+ approvalId: part.approvalId,
408
+ approved: part.approved,
409
+ reason: part.reason,
410
+ providerExecuted: part.providerExecuted,
411
+ ...metadata,
412
+ } satisfies Infer<typeof vToolApprovalResponse>;
413
+ }
311
414
  default:
312
- return part satisfies Infer<typeof vContent>;
415
+ return null;
313
416
  }
314
417
  }),
315
418
  );
316
419
  return {
317
- content: serialized as SerializedContent,
420
+ content: serialized.filter((p) => p !== null) as SerializedContent,
318
421
  fileIds: fileIds.length > 0 ? fileIds : undefined,
319
422
  };
320
423
  }
321
424
 
322
- export function deserializeContent(content: SerializedContent): Content {
425
+ export function fromModelMessageContent(content: Content): Message["content"] {
323
426
  if (typeof content === "string") {
324
427
  return content;
325
428
  }
326
- return content.map((part) => {
327
- switch (part.type) {
328
- case "text":
329
- return {
330
- type: part.type,
331
- text: part.text,
332
- providerOptions: part.providerOptions,
333
- } satisfies TextPart;
334
- case "image":
335
- return {
336
- type: part.type,
337
- image: deserializeUrl(part.image),
338
- mediaType: part.mimeType,
339
- providerOptions: part.providerOptions,
340
- } satisfies ImagePart;
341
- case "file":
342
- return {
343
- type: part.type,
344
- data: deserializeUrl(part.data),
345
- filename: part.filename,
346
- mediaType: part.mimeType,
347
- providerOptions: part.providerOptions,
348
- } satisfies FilePart;
349
- case "tool-call":
350
- return {
351
- type: part.type,
352
- input: part.args ?? null,
353
- providerExecuted: part.providerExecuted,
354
- toolCallId: part.toolCallId,
355
- toolName: part.toolName,
356
- providerOptions: part.providerOptions,
357
- } satisfies ToolCallPart;
358
- case "tool-result":
359
- return {
360
- type: part.type,
361
- output: part.result ?? null,
362
- toolCallId: part.toolCallId,
363
- toolName: part.toolName,
364
- providerOptions: part.providerOptions,
365
- } satisfies ToolResultPart;
366
- case "reasoning":
367
- return {
368
- type: part.type,
369
- text: part.text,
370
- providerMetadata: part.providerOptions,
371
- state: part.state,
372
- } satisfies ReasoningUIPart;
373
- case "redacted-reasoning":
374
- // TODO: should we just drop this?
375
- return {
376
- type: "reasoning",
377
- text: part.data,
378
- providerMetadata: part.providerOptions,
379
- } satisfies ReasoningUIPart;
380
- default:
381
- return part satisfies Content;
382
- }
383
- }) as Content;
429
+ return content
430
+ .map((part) => {
431
+ const metadata: {
432
+ providerOptions?: ProviderOptions;
433
+ providerMetadata?: ProviderMetadata;
434
+ } = {};
435
+ if ("providerOptions" in part) {
436
+ metadata.providerOptions = part.providerOptions as ProviderOptions;
437
+ }
438
+ if ("providerMetadata" in part) {
439
+ metadata.providerMetadata = part.providerMetadata as ProviderMetadata;
440
+ }
441
+ switch (part.type) {
442
+ case "text":
443
+ return part satisfies Infer<typeof vTextPart>;
444
+ case "image":
445
+ return {
446
+ type: part.type,
447
+ mediaType: getMimeOrMediaType(part),
448
+ ...metadata,
449
+ image: serializeDataOrUrl(part.image),
450
+ } satisfies Infer<typeof vImagePart>;
451
+ case "file":
452
+ return {
453
+ type: part.type,
454
+ data: serializeDataOrUrl(part.data),
455
+ filename: part.filename,
456
+ mediaType: getMimeOrMediaType(part)!,
457
+ ...metadata,
458
+ } satisfies Infer<typeof vFilePart>;
459
+ case "tool-call":
460
+ // Handle legacy data where only args field exists
461
+ return {
462
+ type: part.type,
463
+ input: part.input ?? (part as any)?.args ?? {},
464
+ /** @deprecated Use `input` instead. */
465
+ args: part.input ?? (part as any)?.args ?? {},
466
+ toolCallId: part.toolCallId,
467
+ toolName: part.toolName,
468
+ providerExecuted: part.providerExecuted,
469
+ ...metadata,
470
+ } satisfies Infer<typeof vToolCallPart>;
471
+ case "tool-result":
472
+ return normalizeToolResult(part, metadata);
473
+ case "reasoning":
474
+ return {
475
+ type: part.type,
476
+ text: part.text,
477
+ ...metadata,
478
+ } satisfies Infer<typeof vReasoningPart>;
479
+ case "tool-approval-request":
480
+ return {
481
+ type: part.type,
482
+ approvalId: part.approvalId,
483
+ toolCallId: part.toolCallId,
484
+ ...metadata,
485
+ } satisfies Infer<typeof vToolApprovalRequest>;
486
+ case "tool-approval-response":
487
+ return {
488
+ type: part.type,
489
+ approvalId: part.approvalId,
490
+ approved: part.approved,
491
+ reason: part.reason,
492
+ providerExecuted: part.providerExecuted,
493
+ ...metadata,
494
+ } satisfies Infer<typeof vToolApprovalResponse>;
495
+ // Not in current generation output, but could be in historical messages
496
+ default:
497
+ return null;
498
+ }
499
+ })
500
+ .filter((p) => p !== null) as Message["content"];
501
+ }
502
+
503
+ export function toModelMessageContent(
504
+ content: SerializedContent | ModelMessage["content"],
505
+ ): Content {
506
+ if (typeof content === "string") {
507
+ return content;
508
+ }
509
+ return content
510
+ .map((part) => {
511
+ const metadata: {
512
+ providerOptions?: ProviderOptions;
513
+ providerMetadata?: ProviderMetadata;
514
+ } = {};
515
+ if ("providerOptions" in part) {
516
+ metadata.providerOptions = part.providerOptions;
517
+ }
518
+ if ("providerMetadata" in part) {
519
+ metadata.providerMetadata = part.providerMetadata;
520
+ }
521
+ switch (part.type) {
522
+ case "text":
523
+ return {
524
+ type: part.type,
525
+ text: part.text,
526
+ ...metadata,
527
+ } satisfies TextPart;
528
+ case "image":
529
+ return {
530
+ type: part.type,
531
+ image: toModelMessageDataOrUrl(part.image),
532
+ mediaType: getMimeOrMediaType(part),
533
+ ...metadata,
534
+ } satisfies ImagePart;
535
+ case "file":
536
+ return {
537
+ type: part.type,
538
+ data: toModelMessageDataOrUrl(part.data),
539
+ filename: part.filename,
540
+ mediaType: getMimeOrMediaType(part)!,
541
+ ...metadata,
542
+ } satisfies FilePart;
543
+ case "tool-call": {
544
+ // Handle legacy data where only args field exists
545
+ const input = part.input ?? (part as any)?.args ?? {};
546
+ return {
547
+ type: part.type,
548
+ input,
549
+ toolCallId: part.toolCallId,
550
+ toolName: part.toolName,
551
+ providerExecuted: part.providerExecuted,
552
+ ...metadata,
553
+ } satisfies ToolCallPart;
554
+ }
555
+ case "tool-result": {
556
+ return normalizeToolResult(part, metadata);
557
+ }
558
+ case "reasoning":
559
+ return {
560
+ type: part.type,
561
+ text: part.text,
562
+ ...metadata,
563
+ } satisfies ReasoningPart;
564
+ case "redacted-reasoning":
565
+ // TODO: should we just drop this?
566
+ return {
567
+ type: "reasoning",
568
+ text: "",
569
+ ...metadata,
570
+ providerOptions: metadata.providerOptions
571
+ ? {
572
+ ...Object.fromEntries(
573
+ Object.entries(metadata.providerOptions ?? {}).map(
574
+ ([key, value]) => [
575
+ key,
576
+ { ...value, redactedData: part.data },
577
+ ],
578
+ ),
579
+ ),
580
+ }
581
+ : undefined,
582
+ } satisfies ReasoningPart;
583
+ case "source":
584
+ return part satisfies SourcePart;
585
+ case "tool-approval-request":
586
+ return {
587
+ type: part.type,
588
+ approvalId: part.approvalId,
589
+ toolCallId: part.toolCallId,
590
+ ...metadata,
591
+ } satisfies Infer<typeof vToolApprovalRequest>;
592
+ case "tool-approval-response":
593
+ return {
594
+ type: part.type,
595
+ approvalId: part.approvalId,
596
+ approved: part.approved,
597
+ reason: part.reason,
598
+ providerExecuted: part.providerExecuted,
599
+ ...metadata,
600
+ } satisfies Infer<typeof vToolApprovalResponse>;
601
+ default:
602
+ return null;
603
+ }
604
+ })
605
+ .filter((p) => p !== null) as Content;
606
+ }
607
+
608
+ export function normalizeToolOutput(
609
+ result: string | JSONValue | undefined,
610
+ ): ToolResultPart["output"] {
611
+ if (typeof result === "string") {
612
+ return {
613
+ type: "text",
614
+ value: result,
615
+ };
616
+ }
617
+ if (validate(vToolResultOutput, result)) {
618
+ return result;
619
+ }
620
+ return {
621
+ type: "json",
622
+ value: result ?? null,
623
+ };
624
+ }
625
+
626
+ function normalizeToolResult(
627
+ part: ToolResultPart | Infer<typeof vToolResultPart>,
628
+ metadata: {
629
+ providerOptions?: ProviderOptions;
630
+ providerMetadata?: ProviderMetadata;
631
+ },
632
+ ): ToolResultPart & Infer<typeof vToolResultPart> {
633
+ return {
634
+ type: part.type,
635
+ output: part.output
636
+ ? validate(vToolResultOutput, part.output)
637
+ ? (part.output as any)
638
+ : normalizeToolOutput(JSON.stringify(part.output))
639
+ : normalizeToolOutput("result" in part ? part.result : undefined),
640
+ toolCallId: part.toolCallId,
641
+ toolName: part.toolName,
642
+ // Preserve isError flag for error reporting
643
+ ...("isError" in part && part.isError ? { isError: true } : {}),
644
+ ...metadata,
645
+ } satisfies ToolResultPart;
384
646
  }
385
647
 
386
648
  /**
@@ -473,9 +735,12 @@ export function serializeDataOrUrl(
473
735
  ) as ArrayBuffer;
474
736
  }
475
737
 
476
- export function deserializeUrl(
477
- urlOrString: string | ArrayBuffer,
738
+ export function toModelMessageDataOrUrl(
739
+ urlOrString: string | ArrayBuffer | URL | DataContent,
478
740
  ): URL | DataContent {
741
+ if (urlOrString instanceof URL) {
742
+ return urlOrString;
743
+ }
479
744
  if (typeof urlOrString === "string") {
480
745
  if (
481
746
  urlOrString.startsWith("http://") ||
@@ -492,7 +757,7 @@ export function toUIFilePart(part: ImagePart | FilePart): FileUIPart {
492
757
  const dataOrUrl = part.type === "image" ? part.image : part.data;
493
758
  const url =
494
759
  dataOrUrl instanceof ArrayBuffer
495
- ? encodeBase64(dataOrUrl)
760
+ ? convertUint8ArrayToBase64(new Uint8Array(dataOrUrl))
496
761
  : dataOrUrl.toString();
497
762
 
498
763
  return {
@@ -504,10 +769,6 @@ export function toUIFilePart(part: ImagePart | FilePart): FileUIPart {
504
769
  };
505
770
  }
506
771
 
507
- function encodeBase64(data: ArrayBuffer): string {
508
- return Buffer.from(data).toString("base64");
509
- }
510
-
511
772
  // Currently unused
512
773
  // export function toModelMessages(args: {
513
774
  // messages?: ModelMessage[] | AIMessageWithoutId[];