@tanstack/ai 0.36.0 → 0.38.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/dist/esm/activities/chat/adapter.d.ts +163 -0
  2. package/dist/esm/activities/chat/adapter.js +17 -0
  3. package/dist/esm/activities/chat/adapter.js.map +1 -0
  4. package/dist/esm/activities/chat/agent-loop-strategies.d.ts +59 -0
  5. package/dist/esm/activities/chat/agent-loop-strategies.js +23 -0
  6. package/dist/esm/activities/chat/agent-loop-strategies.js.map +1 -0
  7. package/dist/esm/activities/chat/index.d.ts +270 -0
  8. package/dist/esm/activities/chat/index.js +1724 -0
  9. package/dist/esm/activities/chat/index.js.map +1 -0
  10. package/dist/esm/activities/chat/mcp/manager.d.ts +25 -0
  11. package/dist/esm/activities/chat/mcp/manager.js +80 -0
  12. package/dist/esm/activities/chat/mcp/manager.js.map +1 -0
  13. package/dist/esm/activities/chat/mcp/types.d.ts +78 -0
  14. package/dist/esm/activities/chat/messages.d.ts +78 -0
  15. package/dist/esm/activities/chat/messages.js +374 -0
  16. package/dist/esm/activities/chat/messages.js.map +1 -0
  17. package/dist/esm/activities/chat/middleware/builder.d.ts +46 -0
  18. package/dist/esm/activities/chat/middleware/builder.js +17 -0
  19. package/dist/esm/activities/chat/middleware/builder.js.map +1 -0
  20. package/dist/esm/activities/chat/middleware/capabilities.d.ts +93 -0
  21. package/dist/esm/activities/chat/middleware/capabilities.js +45 -0
  22. package/dist/esm/activities/chat/middleware/capabilities.js.map +1 -0
  23. package/dist/esm/activities/chat/middleware/compose.d.ts +87 -0
  24. package/dist/esm/activities/chat/middleware/compose.js +510 -0
  25. package/dist/esm/activities/chat/middleware/compose.js.map +1 -0
  26. package/dist/esm/activities/chat/middleware/define.d.ts +20 -0
  27. package/dist/esm/activities/chat/middleware/define.js +7 -0
  28. package/dist/esm/activities/chat/middleware/define.js.map +1 -0
  29. package/dist/esm/activities/chat/middleware/index.d.ts +10 -0
  30. package/dist/esm/activities/chat/middleware/tool-cache-middleware.d.ts +89 -0
  31. package/dist/esm/activities/chat/middleware/tool-cache-middleware.js +76 -0
  32. package/dist/esm/activities/chat/middleware/tool-cache-middleware.js.map +1 -0
  33. package/dist/esm/activities/chat/middleware/types.d.ts +405 -0
  34. package/dist/esm/activities/chat/middleware/validate.d.ts +19 -0
  35. package/dist/esm/activities/chat/middleware/validate.js +30 -0
  36. package/dist/esm/activities/chat/middleware/validate.js.map +1 -0
  37. package/dist/esm/activities/chat/runtime-context-types.d.ts +43 -0
  38. package/dist/esm/activities/chat/stream/index.d.ts +11 -0
  39. package/dist/esm/activities/chat/stream/json-parser.d.ts +38 -0
  40. package/dist/esm/activities/chat/stream/json-parser.js +28 -0
  41. package/dist/esm/activities/chat/stream/json-parser.js.map +1 -0
  42. package/dist/esm/activities/chat/stream/message-updaters.d.ts +81 -0
  43. package/dist/esm/activities/chat/stream/message-updaters.js +253 -0
  44. package/dist/esm/activities/chat/stream/message-updaters.js.map +1 -0
  45. package/dist/esm/activities/chat/stream/processor.d.ts +439 -0
  46. package/dist/esm/activities/chat/stream/processor.js +1449 -0
  47. package/dist/esm/activities/chat/stream/processor.js.map +1 -0
  48. package/dist/esm/activities/chat/stream/strategies.d.ts +43 -0
  49. package/dist/esm/activities/chat/stream/strategies.js +54 -0
  50. package/dist/esm/activities/chat/stream/strategies.js.map +1 -0
  51. package/dist/esm/activities/chat/stream/types.d.ts +90 -0
  52. package/dist/esm/activities/chat/tools/lazy-tool-manager.d.ts +82 -0
  53. package/dist/esm/activities/chat/tools/lazy-tool-manager.js +194 -0
  54. package/dist/esm/activities/chat/tools/lazy-tool-manager.js.map +1 -0
  55. package/dist/esm/activities/chat/tools/lazy-tools.d.ts +15 -0
  56. package/dist/esm/activities/chat/tools/lazy-tools.js +16 -0
  57. package/dist/esm/activities/chat/tools/lazy-tools.js.map +1 -0
  58. package/dist/esm/activities/chat/tools/schema-converter.d.ts +140 -0
  59. package/dist/esm/activities/chat/tools/schema-converter.js +167 -0
  60. package/dist/esm/activities/chat/tools/schema-converter.js.map +1 -0
  61. package/dist/esm/activities/chat/tools/tool-calls.d.ts +145 -0
  62. package/dist/esm/activities/chat/tools/tool-calls.js +548 -0
  63. package/dist/esm/activities/chat/tools/tool-calls.js.map +1 -0
  64. package/dist/esm/activities/chat/tools/tool-definition.d.ts +135 -0
  65. package/dist/esm/activities/chat/tools/tool-definition.js +25 -0
  66. package/dist/esm/activities/chat/tools/tool-definition.js.map +1 -0
  67. package/dist/esm/activities/error-payload.d.ts +30 -0
  68. package/dist/esm/activities/error-payload.js +54 -0
  69. package/dist/esm/activities/error-payload.js.map +1 -0
  70. package/dist/esm/activities/generateAudio/adapter.d.ts +62 -0
  71. package/dist/esm/activities/generateAudio/adapter.js +16 -0
  72. package/dist/esm/activities/generateAudio/adapter.js.map +1 -0
  73. package/dist/esm/activities/generateAudio/index.d.ts +85 -0
  74. package/dist/esm/activities/generateAudio/index.js +114 -0
  75. package/dist/esm/activities/generateAudio/index.js.map +1 -0
  76. package/dist/esm/activities/generateImage/adapter.d.ts +78 -0
  77. package/dist/esm/activities/generateImage/adapter.js +16 -0
  78. package/dist/esm/activities/generateImage/adapter.js.map +1 -0
  79. package/dist/esm/activities/generateImage/index.d.ts +134 -0
  80. package/dist/esm/activities/generateImage/index.js +121 -0
  81. package/dist/esm/activities/generateImage/index.js.map +1 -0
  82. package/dist/esm/activities/generateSpeech/adapter.d.ts +62 -0
  83. package/dist/esm/activities/generateSpeech/adapter.js +16 -0
  84. package/dist/esm/activities/generateSpeech/adapter.js.map +1 -0
  85. package/dist/esm/activities/generateSpeech/index.d.ts +96 -0
  86. package/dist/esm/activities/generateSpeech/index.js +119 -0
  87. package/dist/esm/activities/generateSpeech/index.js.map +1 -0
  88. package/dist/esm/activities/generateTranscription/adapter.d.ts +62 -0
  89. package/dist/esm/activities/generateTranscription/adapter.js +16 -0
  90. package/dist/esm/activities/generateTranscription/adapter.js.map +1 -0
  91. package/dist/esm/activities/generateTranscription/index.d.ts +109 -0
  92. package/dist/esm/activities/generateTranscription/index.js +109 -0
  93. package/dist/esm/activities/generateTranscription/index.js.map +1 -0
  94. package/dist/esm/activities/generateVideo/adapter.d.ts +145 -0
  95. package/dist/esm/activities/generateVideo/adapter.js +30 -0
  96. package/dist/esm/activities/generateVideo/adapter.js.map +1 -0
  97. package/dist/esm/activities/generateVideo/index.d.ts +223 -0
  98. package/dist/esm/activities/generateVideo/index.js +295 -0
  99. package/dist/esm/activities/generateVideo/index.js.map +1 -0
  100. package/dist/esm/activities/generateVideo/snap.d.ts +14 -0
  101. package/dist/esm/activities/generateVideo/snap.js +54 -0
  102. package/dist/esm/activities/generateVideo/snap.js.map +1 -0
  103. package/dist/esm/activities/index.d.ts +27 -0
  104. package/dist/esm/activities/index.js +43 -0
  105. package/dist/esm/activities/index.js.map +1 -0
  106. package/dist/esm/activities/middleware/index.d.ts +2 -0
  107. package/dist/esm/activities/middleware/run.d.ts +20 -0
  108. package/dist/esm/activities/middleware/run.js +42 -0
  109. package/dist/esm/activities/middleware/run.js.map +1 -0
  110. package/dist/esm/activities/middleware/types.d.ts +118 -0
  111. package/dist/esm/activities/stream-generation-result.d.ts +15 -0
  112. package/dist/esm/activities/stream-generation-result.js +49 -0
  113. package/dist/esm/activities/stream-generation-result.js.map +1 -0
  114. package/dist/esm/activities/summarize/adapter.d.ts +74 -0
  115. package/dist/esm/activities/summarize/adapter.js +16 -0
  116. package/dist/esm/activities/summarize/adapter.js.map +1 -0
  117. package/dist/esm/activities/summarize/chat-stream-summarize.d.ts +45 -0
  118. package/dist/esm/activities/summarize/chat-stream-summarize.js +212 -0
  119. package/dist/esm/activities/summarize/chat-stream-summarize.js.map +1 -0
  120. package/dist/esm/activities/summarize/index.d.ts +108 -0
  121. package/dist/esm/activities/summarize/index.js +113 -0
  122. package/dist/esm/activities/summarize/index.js.map +1 -0
  123. package/dist/esm/adapter-internals.d.ts +5 -0
  124. package/dist/esm/adapter-internals.js +10 -0
  125. package/dist/esm/adapter-internals.js.map +1 -0
  126. package/dist/esm/client.d.ts +44 -0
  127. package/dist/esm/client.js +67 -0
  128. package/dist/esm/client.js.map +1 -0
  129. package/dist/esm/extend-adapter.d.ts +152 -0
  130. package/dist/esm/extend-adapter.js +21 -0
  131. package/dist/esm/extend-adapter.js.map +1 -0
  132. package/dist/esm/index.d.ts +45 -0
  133. package/dist/esm/index.js +109 -0
  134. package/dist/esm/index.js.map +1 -0
  135. package/dist/esm/logger/console-logger.d.ts +29 -0
  136. package/dist/esm/logger/console-logger.js +83 -0
  137. package/dist/esm/logger/console-logger.js.map +1 -0
  138. package/dist/esm/logger/internal-logger.d.ts +41 -0
  139. package/dist/esm/logger/internal-logger.js +86 -0
  140. package/dist/esm/logger/internal-logger.js.map +1 -0
  141. package/dist/esm/logger/resolve.d.ts +14 -0
  142. package/dist/esm/logger/resolve.js +54 -0
  143. package/dist/esm/logger/resolve.js.map +1 -0
  144. package/dist/esm/logger/types.d.ts +75 -0
  145. package/dist/esm/middlewares/content-guard.d.ts +77 -0
  146. package/dist/esm/middlewares/content-guard.js +156 -0
  147. package/dist/esm/middlewares/content-guard.js.map +1 -0
  148. package/dist/esm/middlewares/index.d.ts +2 -0
  149. package/dist/esm/middlewares/index.js +7 -0
  150. package/dist/esm/middlewares/index.js.map +1 -0
  151. package/dist/esm/middlewares/otel.d.ts +81 -0
  152. package/dist/esm/middlewares/otel.js +745 -0
  153. package/dist/esm/middlewares/otel.js.map +1 -0
  154. package/dist/esm/middlewares/tool-cache.d.ts +1 -0
  155. package/dist/esm/middlewares/usage-attributes.d.ts +24 -0
  156. package/dist/esm/middlewares/usage-attributes.js +43 -0
  157. package/dist/esm/middlewares/usage-attributes.js.map +1 -0
  158. package/dist/esm/realtime/index.d.ts +28 -0
  159. package/dist/esm/realtime/index.js +8 -0
  160. package/dist/esm/realtime/index.js.map +1 -0
  161. package/dist/esm/realtime/types.d.ts +282 -0
  162. package/dist/esm/stream-to-response.d.ts +102 -0
  163. package/dist/esm/stream-to-response.js +121 -0
  164. package/dist/esm/stream-to-response.js.map +1 -0
  165. package/dist/esm/strip-to-spec-middleware.d.ts +18 -0
  166. package/dist/esm/strip-to-spec-middleware.js +20 -0
  167. package/dist/esm/strip-to-spec-middleware.js.map +1 -0
  168. package/dist/esm/system-prompts.d.ts +66 -0
  169. package/dist/esm/system-prompts.js +23 -0
  170. package/dist/esm/system-prompts.js.map +1 -0
  171. package/dist/esm/tool-registry.d.ts +81 -0
  172. package/dist/esm/tool-registry.js +49 -0
  173. package/dist/esm/tool-registry.js.map +1 -0
  174. package/dist/esm/tools/provider-tool.d.ts +30 -0
  175. package/dist/esm/tools/provider-tool.js +7 -0
  176. package/dist/esm/tools/provider-tool.js.map +1 -0
  177. package/dist/esm/types.d.ts +1627 -0
  178. package/dist/esm/utilities/ag-ui-wire.d.ts +44 -0
  179. package/dist/esm/utilities/ag-ui-wire.js +107 -0
  180. package/dist/esm/utilities/ag-ui-wire.js.map +1 -0
  181. package/dist/esm/utilities/chat-params.d.ts +85 -0
  182. package/dist/esm/utilities/chat-params.js +100 -0
  183. package/dist/esm/utilities/chat-params.js.map +1 -0
  184. package/dist/esm/utilities/errors.d.ts +13 -0
  185. package/dist/esm/utilities/errors.js +22 -0
  186. package/dist/esm/utilities/errors.js.map +1 -0
  187. package/dist/esm/utilities/media-prompt.d.ts +35 -0
  188. package/dist/esm/utilities/media-prompt.js +43 -0
  189. package/dist/esm/utilities/media-prompt.js.map +1 -0
  190. package/dist/esm/utilities/numbers.d.ts +8 -0
  191. package/dist/esm/utilities/numbers.js +12 -0
  192. package/dist/esm/utilities/numbers.js.map +1 -0
  193. package/dist/esm/utilities/sampling-keys.d.ts +20 -0
  194. package/dist/esm/utilities/sampling-keys.js +20 -0
  195. package/dist/esm/utilities/sampling-keys.js.map +1 -0
  196. package/dist/esm/utilities/tool-result.d.ts +21 -0
  197. package/dist/esm/utilities/tool-result.js +37 -0
  198. package/dist/esm/utilities/tool-result.js.map +1 -0
  199. package/dist/esm/utilities/usage.d.ts +31 -0
  200. package/dist/esm/utilities/usage.js +11 -0
  201. package/dist/esm/utilities/usage.js.map +1 -0
  202. package/dist/esm/utils.d.ts +17 -0
  203. package/dist/esm/utils.js +20 -0
  204. package/dist/esm/utils.js.map +1 -0
  205. package/package.json +3 -3
  206. package/skills/ai-core/chat-experience/SKILL.md +30 -2
  207. package/skills/ai-core/media-generation/SKILL.md +13 -0
  208. package/skills/ai-core/tool-calling/SKILL.md +10 -3
  209. package/src/activities/chat/mcp/manager.ts +31 -1
  210. package/src/activities/chat/mcp/types.ts +23 -0
  211. package/src/activities/chat/messages.ts +5 -0
  212. package/src/activities/chat/stream/processor.ts +42 -0
  213. package/src/activities/chat/tools/tool-calls.ts +96 -2
  214. package/src/client.ts +1 -0
  215. package/src/types.ts +31 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tool-result.js","sources":["../../../src/utilities/tool-result.ts"],"sourcesContent":["import type { ContentPart } from '../types'\n\nconst CONTENT_PART_TYPES = new Set([\n 'text',\n 'image',\n 'audio',\n 'video',\n 'document',\n])\n\n/**\n * Structural check for a single `ContentPart`. A text part must carry a string\n * `content`; every other modality must carry a `source` with `type` of\n * `'url' | 'data'` and a string `value`.\n */\nexport function isContentPart(value: unknown): value is ContentPart {\n if (typeof value !== 'object' || value === null) return false\n const part = value as Record<string, unknown>\n if (typeof part.type !== 'string' || !CONTENT_PART_TYPES.has(part.type)) {\n return false\n }\n if (part.type === 'text') {\n return typeof part.content === 'string'\n }\n const source = part.source\n if (typeof source !== 'object' || source === null) return false\n const src = source as Record<string, unknown>\n if (typeof src.value !== 'string') return false\n // `data` sources require a mimeType (matches ContentPartDataSource); `url`\n // sources don't. Requiring it here keeps the runtime guard consistent with\n // the type and avoids emitting `data:undefined;base64,...` downstream.\n if (src.type === 'data') return typeof src.mimeType === 'string'\n return src.type === 'url'\n}\n\n/**\n * True iff `value` is a NON-EMPTY array whose every element is a valid\n * `ContentPart`. Empty arrays and mixed arrays return false so they continue\n * to be treated as ordinary (stringified) data — this keeps the auto-detection\n * footgun narrow.\n */\nexport function isContentPartArray(\n value: unknown,\n): value is Array<ContentPart> {\n return Array.isArray(value) && value.length > 0 && value.every(isContentPart)\n}\n\n/**\n * Normalize a tool's return value for transport:\n * - string → unchanged\n * - ContentPart array → unchanged (multimodal, passed through to the adapter)\n * - anything else → `JSON.stringify`\n */\nexport function normalizeToolResult(\n result: unknown,\n): string | Array<ContentPart> {\n if (typeof result === 'string') return result\n if (isContentPartArray(result)) return result\n return JSON.stringify(result)\n}\n"],"names":[],"mappings":"AAEA,MAAM,yCAAyB,IAAI;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAOM,SAAS,cAAc,OAAsC;AAClE,MAAI,OAAO,UAAU,YAAY,UAAU,KAAM,QAAO;AACxD,QAAM,OAAO;AACb,MAAI,OAAO,KAAK,SAAS,YAAY,CAAC,mBAAmB,IAAI,KAAK,IAAI,GAAG;AACvE,WAAO;AAAA,EACT;AACA,MAAI,KAAK,SAAS,QAAQ;AACxB,WAAO,OAAO,KAAK,YAAY;AAAA,EACjC;AACA,QAAM,SAAS,KAAK;AACpB,MAAI,OAAO,WAAW,YAAY,WAAW,KAAM,QAAO;AAC1D,QAAM,MAAM;AACZ,MAAI,OAAO,IAAI,UAAU,SAAU,QAAO;AAI1C,MAAI,IAAI,SAAS,OAAQ,QAAO,OAAO,IAAI,aAAa;AACxD,SAAO,IAAI,SAAS;AACtB;AAQO,SAAS,mBACd,OAC6B;AAC7B,SAAO,MAAM,QAAQ,KAAK,KAAK,MAAM,SAAS,KAAK,MAAM,MAAM,aAAa;AAC9E;AAQO,SAAS,oBACd,QAC6B;AAC7B,MAAI,OAAO,WAAW,SAAU,QAAO;AACvC,MAAI,mBAAmB,MAAM,EAAG,QAAO;AACvC,SAAO,KAAK,UAAU,MAAM;AAC9B;"}
@@ -0,0 +1,31 @@
1
+ import { ProviderUsageDetails, TokenUsage } from '../types.js';
2
+ /**
3
+ * Input parameters for building base TokenUsage.
4
+ * Provider functions should extract these from their SDK's response.
5
+ */
6
+ export interface BaseUsageInput {
7
+ /** Total input/prompt tokens */
8
+ promptTokens: number;
9
+ /** Total output/completion tokens */
10
+ completionTokens: number;
11
+ /** Total tokens (prompt + completion) */
12
+ totalTokens: number;
13
+ }
14
+ /**
15
+ * Builds the base TokenUsage object with core fields.
16
+ * Provider-specific functions should use this and then add their own details.
17
+ *
18
+ * @param input - The base token counts
19
+ * @returns A TokenUsage object with promptTokens, completionTokens, totalTokens
20
+ *
21
+ * @example
22
+ * ```typescript
23
+ * const base = buildBaseUsage({
24
+ * promptTokens: 100,
25
+ * completionTokens: 50,
26
+ * totalTokens: 150
27
+ * });
28
+ * // Returns: { promptTokens: 100, completionTokens: 50, totalTokens: 150 }
29
+ * ```
30
+ */
31
+ export declare function buildBaseUsage<TProviderDetails = ProviderUsageDetails>(input: BaseUsageInput): TokenUsage<TProviderDetails>;
@@ -0,0 +1,11 @@
1
+ function buildBaseUsage(input) {
2
+ return {
3
+ promptTokens: input.promptTokens,
4
+ completionTokens: input.completionTokens,
5
+ totalTokens: input.totalTokens
6
+ };
7
+ }
8
+ export {
9
+ buildBaseUsage
10
+ };
11
+ //# sourceMappingURL=usage.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"usage.js","sources":["../../../src/utilities/usage.ts"],"sourcesContent":["import type { ProviderUsageDetails, TokenUsage } from '../types'\n\n/**\n * Input parameters for building base TokenUsage.\n * Provider functions should extract these from their SDK's response.\n */\nexport interface BaseUsageInput {\n /** Total input/prompt tokens */\n promptTokens: number\n /** Total output/completion tokens */\n completionTokens: number\n /** Total tokens (prompt + completion) */\n totalTokens: number\n}\n\n/**\n * Builds the base TokenUsage object with core fields.\n * Provider-specific functions should use this and then add their own details.\n *\n * @param input - The base token counts\n * @returns A TokenUsage object with promptTokens, completionTokens, totalTokens\n *\n * @example\n * ```typescript\n * const base = buildBaseUsage({\n * promptTokens: 100,\n * completionTokens: 50,\n * totalTokens: 150\n * });\n * // Returns: { promptTokens: 100, completionTokens: 50, totalTokens: 150 }\n * ```\n */\nexport function buildBaseUsage<TProviderDetails = ProviderUsageDetails>(\n input: BaseUsageInput,\n): TokenUsage<TProviderDetails> {\n return {\n promptTokens: input.promptTokens,\n completionTokens: input.completionTokens,\n totalTokens: input.totalTokens,\n }\n}\n"],"names":[],"mappings":"AAgCO,SAAS,eACd,OAC8B;AAC9B,SAAO;AAAA,IACL,cAAc,MAAM;AAAA,IACpB,kBAAkB,MAAM;AAAA,IACxB,aAAa,MAAM;AAAA,EAAA;AAEvB;"}
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Detect image mime type from base64 data using magic bytes.
3
+ * Returns undefined if the format cannot be detected.
4
+ *
5
+ * This function analyzes the first few bytes of base64-encoded image data
6
+ * to determine the image format based on file signature (magic bytes).
7
+ *
8
+ * @param base64Data - The base64-encoded image data
9
+ * @returns The detected mime type, or undefined if unrecognized
10
+ *
11
+ * @example
12
+ * ```ts
13
+ * const mimeType = detectImageMimeType(imageBase64)
14
+ * // Returns 'image/jpeg', 'image/png', 'image/gif', 'image/webp', or undefined
15
+ * ```
16
+ */
17
+ export declare function detectImageMimeType(base64Data: string): 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp' | undefined;
@@ -0,0 +1,20 @@
1
+ function detectImageMimeType(base64Data) {
2
+ const prefix = base64Data.substring(0, 20);
3
+ if (prefix.startsWith("/9j/")) {
4
+ return "image/jpeg";
5
+ }
6
+ if (prefix.startsWith("iVBORw0KGgo")) {
7
+ return "image/png";
8
+ }
9
+ if (prefix.startsWith("R0lGOD")) {
10
+ return "image/gif";
11
+ }
12
+ if (prefix.startsWith("UklGR")) {
13
+ return "image/webp";
14
+ }
15
+ return void 0;
16
+ }
17
+ export {
18
+ detectImageMimeType
19
+ };
20
+ //# sourceMappingURL=utils.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utils.js","sources":["../../src/utils.ts"],"sourcesContent":["/**\n * Detect image mime type from base64 data using magic bytes.\n * Returns undefined if the format cannot be detected.\n *\n * This function analyzes the first few bytes of base64-encoded image data\n * to determine the image format based on file signature (magic bytes).\n *\n * @param base64Data - The base64-encoded image data\n * @returns The detected mime type, or undefined if unrecognized\n *\n * @example\n * ```ts\n * const mimeType = detectImageMimeType(imageBase64)\n * // Returns 'image/jpeg', 'image/png', 'image/gif', 'image/webp', or undefined\n * ```\n */\nexport function detectImageMimeType(\n base64Data: string,\n): 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp' | undefined {\n // Get first few bytes (base64 encoded)\n const prefix = base64Data.substring(0, 20)\n\n // JPEG: starts with /9j/ (FFD8FF in base64)\n if (prefix.startsWith('/9j/')) {\n return 'image/jpeg'\n }\n // PNG: starts with iVBORw0KGgo (89504E47 in base64)\n if (prefix.startsWith('iVBORw0KGgo')) {\n return 'image/png'\n }\n // GIF: starts with R0lGOD (474946 in base64)\n if (prefix.startsWith('R0lGOD')) {\n return 'image/gif'\n }\n // WebP: starts with UklGR (52494646 in base64, followed by WEBP)\n if (prefix.startsWith('UklGR')) {\n return 'image/webp'\n }\n\n return undefined\n}\n"],"names":[],"mappings":"AAgBO,SAAS,oBACd,YACqE;AAErE,QAAM,SAAS,WAAW,UAAU,GAAG,EAAE;AAGzC,MAAI,OAAO,WAAW,MAAM,GAAG;AAC7B,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,WAAW,aAAa,GAAG;AACpC,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,WAAW,QAAQ,GAAG;AAC/B,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,WAAW,OAAO,GAAG;AAC9B,WAAO;AAAA,EACT;AAEA,SAAO;AACT;"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tanstack/ai",
3
- "version": "0.36.0",
3
+ "version": "0.38.0",
4
4
  "description": "Type-safe TypeScript AI SDK for streaming chat, tool calling, agents, structured outputs, and multimodal generation.",
5
5
  "author": "Tanner Linsley",
6
6
  "license": "MIT",
@@ -76,8 +76,8 @@
76
76
  "@ag-ui/core": "^0.0.52",
77
77
  "@standard-schema/spec": "^1.1.0",
78
78
  "partial-json": "^0.1.7",
79
- "@tanstack/ai-event-client": "0.6.7",
80
- "@tanstack/ai-utils": "0.3.0"
79
+ "@tanstack/ai-event-client": "0.6.8",
80
+ "@tanstack/ai-utils": "0.3.1"
81
81
  },
82
82
  "peerDependencies": {
83
83
  "@opentelemetry/api": ">=1.9.0"
@@ -278,7 +278,35 @@ if (part.type === 'image') {
278
278
  }
279
279
  ```
280
280
 
281
- ### 4. HTTP Stream Format (Alternative to SSE)
281
+ ### 4. Sending Audio Messages (Browser Recording)
282
+
283
+ Use `useAudioRecorder` from `@tanstack/ai-react` (or `createAudioRecorder` in Svelte) to capture audio in the browser. The resolved `AudioRecording` includes a ready-to-use `part` that slots directly into `sendMessage`.
284
+
285
+ ```typescript
286
+ import {
287
+ useAudioRecorder,
288
+ useChat,
289
+ fetchServerSentEvents,
290
+ } from '@tanstack/ai-react'
291
+
292
+ const { isRecording, isSupported, start, stop } = useAudioRecorder()
293
+ const { sendMessage } = useChat({
294
+ connection: fetchServerSentEvents('/api/chat'),
295
+ })
296
+
297
+ async function toggle() {
298
+ if (!isRecording) {
299
+ await start()
300
+ return
301
+ }
302
+ const recording = await stop()
303
+ await sendMessage({ content: [recording.part] })
304
+ }
305
+ ```
306
+
307
+ `recording.part` is `{ type: 'audio', source: { type: 'data', value: base64, mimeType } }`. Returns the recorder's native format (`audio/webm` or `audio/mp4`) with no transcoding.
308
+
309
+ ### 5. HTTP Stream Format (Alternative to SSE)
282
310
 
283
311
  Use `toHttpResponse` + `fetchHttpStream` for newline-delimited JSON instead of SSE.
284
312
 
@@ -310,7 +338,7 @@ const { messages, sendMessage } = useChat({
310
338
  The only difference is swapping `toServerSentEventsResponse` / `fetchServerSentEvents`
311
339
  for `toHttpResponse` / `fetchHttpStream`. Everything else stays identical.
312
340
 
313
- ### 5. MCP Tool Discovery via `chat({ mcp })`
341
+ ### 6. MCP Tool Discovery via `chat({ mcp })`
314
342
 
315
343
  Pass `mcp` to let `chat()` own discovery **and** lifecycle for one or more MCP
316
344
  clients. Useful when you want minimal boilerplate and don't need to reuse the
@@ -359,6 +359,19 @@ const { generate, result, isLoading } = useGenerateSpeech({
359
359
  Adapter: `openaiTranscription` (whisper-1, gpt-4o-transcribe,
360
360
  gpt-4o-mini-transcribe).
361
361
 
362
+ > **Capturing audio in the browser:** Use `useAudioRecorder` from `@tanstack/ai-react` to record directly in the browser, then pass the recording as the `audio` input to `generate()`, or use `recording.part` as a prompt part in chat/generation calls. No transcoding or extra dependencies required — the recorder returns the native browser format (`audio/webm` or `audio/mp4`). For transcription, wrap it as a `data:` URL so the provider gets the real content type; passing raw `recording.base64` makes the adapter assume `audio/mpeg` and mislabel the webm/mp4 bytes.
363
+ >
364
+ > ```typescript
365
+ > const { isRecording, start, stop } = useAudioRecorder()
366
+ > const { generate } = useTranscription({
367
+ > connection: fetchServerSentEvents('/api/transcribe'),
368
+ > })
369
+ > // ...
370
+ > const recording = await stop()
371
+ > const mimeType = recording.mimeType.split(';')[0] // strip ;codecs=...
372
+ > await generate({ audio: `data:${mimeType};base64,${recording.base64}` })
373
+ > ```
374
+
362
375
  ```typescript
363
376
  import { generateTranscription } from '@tanstack/ai'
364
377
  import { openaiTranscription } from '@tanstack/ai-openai'
@@ -75,7 +75,7 @@ import { updateCartUIDef } from '@/tools/definitions'
75
75
  export async function POST(request: Request) {
76
76
  const { messages } = await request.json()
77
77
  const stream = chat({
78
- adapter: openaiText('gpt-4o'),
78
+ adapter: openaiText('gpt-5.5'),
79
79
  messages,
80
80
  tools: [getProducts, updateCartUIDef], // server tool + client definition
81
81
  })
@@ -158,7 +158,7 @@ const getUserData = getUserDataDef.server(async ({ userId }) => {
158
158
 
159
159
  // In your route handler:
160
160
  const stream = chat({
161
- adapter: openaiText('gpt-4o'),
161
+ adapter: openaiText('gpt-5.5'),
162
162
  messages,
163
163
  tools: [getUserData],
164
164
  })
@@ -188,7 +188,7 @@ Server -- pass definition only (no execute function):
188
188
 
189
189
  ```typescript
190
190
  const stream = chat({
191
- adapter: openaiText('gpt-4o'),
191
+ adapter: openaiText('gpt-5.5'),
192
192
  messages,
193
193
  tools: [showNotificationDef],
194
194
  })
@@ -405,6 +405,13 @@ The post-discovery payload always returns the full description and schema regard
405
405
  `@tanstack/ai-mcp` lets a server-side `chat()` call discover and invoke tools
406
406
  hosted on any MCP server (Streamable HTTP, SSE, or stdio).
407
407
 
408
+ **MCP tools and UI resources:** When an MCP tool result carries a `ui://`
409
+ resource URI (via `_meta.ui.resourceUri`), TanStack AI surfaces it as a
410
+ `UIResourcePart` on the assistant `UIMessage` in the client message list.
411
+ `UIResourcePart` is a presentational-only part — it never enters model input.
412
+ See the `@tanstack/ai-mcp` skill for the full MCP Apps API
413
+ (`createMcpAppCallHandler`, `createMcpAppBridge`, `MCPAppResource`).
414
+
408
415
  ### Basic usage — auto-discovery
409
416
 
410
417
  ```typescript
@@ -1,6 +1,33 @@
1
1
  import type { ServerTool } from '../tools/tool-definition'
2
2
  import type { ChatMCPOptions, MCPToolSource } from './types'
3
3
 
4
+ /**
5
+ * Bind the source's `readResource` onto a ui-linked tool's `metadata.mcp` so it
6
+ * travels with the tool to the server-tool execution/emit site (`tool-calls.ts`).
7
+ *
8
+ * `discover()` is the single place in `@tanstack/ai` that has both a tool and
9
+ * its originating source, and `@tanstack/ai` must not import `@tanstack/ai-mcp`,
10
+ * so this is where the source handle is threaded onto the tool. Only tools that
11
+ * actually link a `ui://` resource (their discovery stamped
12
+ * `metadata.mcp.uiResourceUri`) and whose source can read resources get bound;
13
+ * everything else is left untouched.
14
+ *
15
+ * This mutates the discovered tool's `metadata.mcp` in place. That is safe
16
+ * because discovery returns fresh tool objects per `discover()` call: the bound
17
+ * `readResource` closes over `source`, whose connection stays live until the run
18
+ * drains. If discovery results were ever cached and reused across runs, this
19
+ * would bind a closure over an already-closed source — bind onto a copy then.
20
+ */
21
+ function bindReadResource(tool: ServerTool, source: MCPToolSource): void {
22
+ if (!source.readResource) return
23
+ const meta = (
24
+ tool.metadata as { mcp?: { uiResourceUri?: string } } | undefined
25
+ )?.mcp
26
+ if (!meta?.uiResourceUri) return
27
+ ;(meta as { readResource?: MCPToolSource['readResource'] }).readResource =
28
+ source.readResource.bind(source)
29
+ }
30
+
4
31
  export class MCPDuplicateToolNameError extends Error {
5
32
  constructor(public readonly toolName: string) {
6
33
  super(
@@ -57,7 +84,10 @@ export class MCPManager {
57
84
  for (const [source, result] of zipped) {
58
85
  if (result === undefined) continue
59
86
  if (result.status === 'fulfilled') {
60
- tools.push(...result.value)
87
+ for (const t of result.value) {
88
+ bindReadResource(t, source)
89
+ tools.push(t)
90
+ }
61
91
  } else if (this.#onDiscoveryError) {
62
92
  // throw/reject inside handler ⇒ propagate (fail-fast); return ⇒ skip
63
93
  await this.#onDiscoveryError(result.reason, source)
@@ -1,5 +1,20 @@
1
1
  import type { ServerTool } from '../tools/tool-definition'
2
2
 
3
+ /**
4
+ * The shape `readResource` resolves to — a structural subset of MCP's
5
+ * `ReadResourceResult`. Single source of truth shared by
6
+ * `MCPToolSource.readResource` (this file) and the tool-bound
7
+ * `McpToolAppMeta.readResource` (tool-calls.ts) so the two copies cannot drift.
8
+ */
9
+ export interface McpResourceReadResult {
10
+ contents: Array<{
11
+ uri: string
12
+ mimeType?: string
13
+ text?: string
14
+ blob?: string
15
+ }>
16
+ }
17
+
3
18
  /**
4
19
  * Minimal structural shape that `chat({ mcp })` needs from an MCP client.
5
20
  *
@@ -13,6 +28,14 @@ export interface MCPToolSource {
13
28
  // forwards what is declared here.
14
29
  tools: (options?: { lazy?: boolean }) => Promise<Array<ServerTool>>
15
30
  close: () => Promise<void>
31
+ /**
32
+ * Reads an MCP resource by URI. Used by the chat manager to eagerly fetch
33
+ * `ui://` resource widgets (MCP Apps) after a tool result resolves.
34
+ *
35
+ * Optional — sources that do not serve `ui://` resources need not implement
36
+ * this method. `ai-mcp`'s `MCPClient` satisfies this structurally.
37
+ */
38
+ readResource?: (uri: string) => Promise<McpResourceReadResult>
16
39
  }
17
40
 
18
41
  /**
@@ -322,6 +322,11 @@ function buildAssistantMessages(uiMessage: UIMessage): Array<ModelMessage> {
322
322
  }
323
323
  break
324
324
 
325
+ case 'ui-resource':
326
+ // MCP Apps widget — rendered client-side only. It must never enter
327
+ // model input, so it is intentionally dropped from the model message.
328
+ break
329
+
325
330
  default:
326
331
  break
327
332
  }
@@ -56,6 +56,8 @@ import type {
56
56
  ToolCallPart,
57
57
  ToolResultPart,
58
58
  UIMessage,
59
+ UIResourceEvent,
60
+ UIResourcePart,
59
61
  } from '../../../types'
60
62
 
61
63
  /**
@@ -1624,6 +1626,46 @@ export class StreamProcessor {
1624
1626
  return
1625
1627
  }
1626
1628
 
1629
+ // Handle MCP Apps ui-resource events — materialize a UIResourcePart on the
1630
+ // active assistant message. Never falls through to onCustomEvent because
1631
+ // ui-resource is a system event, not a user-defined custom event.
1632
+ if (chunk.name === 'ui-resource' && chunk.value) {
1633
+ const v: UIResourceEvent['value'] = chunk.value
1634
+ // Resolve the target assistant message. When a toolCallId is present, the
1635
+ // tool call's OWNER message is authoritative, so prefer it first; fall
1636
+ // back to the active assistant id only if the tool call isn't mapped.
1637
+ // This avoids misattaching the widget to a different active message in a
1638
+ // multi-message session.
1639
+ const resolvedMessageId =
1640
+ this.toolCallToMessage.get(v.toolCallId) ?? messageId
1641
+ if (resolvedMessageId) {
1642
+ const part: UIResourcePart = {
1643
+ type: 'ui-resource',
1644
+ resource: v.resource,
1645
+ toolCallId: v.toolCallId,
1646
+ toolName: v.toolName,
1647
+ ...(v.serverId !== undefined && { serverId: v.serverId }),
1648
+ ...(v.meta !== undefined && { meta: v.meta }),
1649
+ }
1650
+ this.messages = this.messages.map((msg) =>
1651
+ msg.id === resolvedMessageId
1652
+ ? { ...msg, parts: [...msg.parts, part] }
1653
+ : msg,
1654
+ )
1655
+ this.emitMessagesChange()
1656
+ } else {
1657
+ // No owner message and no active assistant id — the server read and
1658
+ // streamed a widget that has nowhere to attach (e.g. a toolCallId never
1659
+ // registered, or the event arrived after the run cleared its active
1660
+ // ids). Drop fail-soft, but warn: a vanished widget is otherwise
1661
+ // undebuggable from the client.
1662
+ console.warn(
1663
+ `[mcp-apps] dropped ui-resource: no target message for toolCallId "${v.toolCallId}" (toolName "${v.toolName}")`,
1664
+ )
1665
+ }
1666
+ return
1667
+ }
1668
+
1627
1669
  // Forward non-system custom events to onCustomEvent callback
1628
1670
  if (this.events.onCustomEvent) {
1629
1671
  const toolCallId =
@@ -18,6 +18,7 @@ import type {
18
18
  AfterToolCallInfo,
19
19
  BeforeToolCallDecision,
20
20
  } from '../middleware/types'
21
+ import type { McpResourceReadResult } from '../mcp/types'
21
22
  import type {
22
23
  ContextFromTool,
23
24
  DefinedContext,
@@ -33,6 +34,92 @@ function safeJsonParse(value: string): unknown {
33
34
  }
34
35
  }
35
36
 
37
+ /**
38
+ * MCP Apps metadata attached to a server tool at discovery (see
39
+ * `@tanstack/ai-mcp` discovery + `MCPManager.discover()`).
40
+ *
41
+ * - `uiResourceUri` / `serverId` are stamped by ai-mcp at tool discovery.
42
+ * - `readResource` is bound by `MCPManager.discover()` (the one site that has
43
+ * both the tool and its originating source) so the resource can be eagerly
44
+ * read at the emit site. Under `chat()`-managed MCP lifecycle
45
+ * (`connection:'close'`), the MCP source is not disposed until the run
46
+ * drains, so `readResource` is still live at this emit point. Note: a caller
47
+ * who closes the MCP source early (outside `chat()`'s managed lifecycle)
48
+ * degrades fail-soft — `readResource` may reject, the widget is absent, but
49
+ * the tool result still flows to the model.
50
+ * `@tanstack/ai` never imports `@tanstack/ai-mcp`; this travels structurally
51
+ * on the tool.
52
+ */
53
+ interface McpToolAppMeta {
54
+ uiResourceUri?: string
55
+ serverId?: string
56
+ /** Server-native (unprefixed) MCP tool name — used as the renderer's toolName. */
57
+ serverToolName?: string
58
+ readResource?: (uri: string) => Promise<McpResourceReadResult>
59
+ }
60
+
61
+ function readMcpAppMeta(tool: AnyTool): McpToolAppMeta | undefined {
62
+ const meta = (tool.metadata as { mcp?: McpToolAppMeta } | undefined)?.mcp
63
+ return meta
64
+ }
65
+
66
+ /**
67
+ * Eagerly read a tool's linked `ui://` resource (MCP Apps) and emit a
68
+ * `ui-resource` CUSTOM event so the client can render the widget. The model
69
+ * still receives the normal text tool-result; the widget rides alongside and
70
+ * never enters model input.
71
+ *
72
+ * Fail-soft: any read error logs a warning and emits nothing — it never throws,
73
+ * so the normal tool-result still flows and a broken widget cannot break the run.
74
+ */
75
+ async function emitUiResourceIfLinked<TContext>(
76
+ tool: AnyTool,
77
+ context: ToolExecutionContext<TContext>,
78
+ ): Promise<void> {
79
+ const mcp = readMcpAppMeta(tool)
80
+ const uiUri = mcp?.uiResourceUri
81
+ if (!uiUri || !mcp.readResource) return
82
+
83
+ // The try covers ONLY the fallible read — keep `emitCustomEvent` out of it so
84
+ // an exception from the emit path can't be mislabeled as a read failure.
85
+ let matched: McpResourceReadResult['contents'][number] | undefined
86
+ try {
87
+ const res = await mcp.readResource(uiUri)
88
+ // Emit ONLY the content whose uri matches the requested `uiUri`. A source
89
+ // can return unrelated contents; falling back to `contents[0]` would risk
90
+ // rendering a widget that doesn't correspond to the linked resource. This
91
+ // is a display widget — a mismatched resource is worse than none, so if no
92
+ // content matches we fail-soft (warn + return) rather than emit.
93
+ matched = res.contents.find((c) => c.uri === uiUri)
94
+ } catch (err) {
95
+ // fail-soft — the text tool-result already flows; a broken widget must
96
+ // not break the run.
97
+ console.warn(`[mcp-apps] failed to read ui resource ${uiUri}:`, err)
98
+ return
99
+ }
100
+ if (!matched) {
101
+ console.warn(
102
+ `[mcp-apps] ui resource ${uiUri} returned no content matching that uri; not emitting`,
103
+ )
104
+ return
105
+ }
106
+ // NOTE: `toolCallId` is intentionally NOT set here — it is stamped onto
107
+ // every emitted event by the `executeToolCalls` context wrapper, so the
108
+ // UIResourceEvent.value.toolCallId / UIResourcePart.toolCallId contract is
109
+ // still satisfied downstream.
110
+ context.emitCustomEvent('ui-resource', {
111
+ resource: {
112
+ uri: matched.uri,
113
+ mimeType: matched.mimeType ?? 'text/html',
114
+ text: matched.text,
115
+ blob: matched.blob,
116
+ },
117
+ serverId: mcp.serverId,
118
+ toolName: mcp.serverToolName ?? tool.name,
119
+ meta: undefined,
120
+ })
121
+ }
122
+
36
123
  /**
37
124
  * Optional middleware hooks for tool execution.
38
125
  * When provided, these callbacks are invoked before/after each tool execution.
@@ -456,7 +543,7 @@ async function applyBeforeToolCallDecision(
456
543
  * Execute a server-side tool with event polling, output validation, and middleware hooks.
457
544
  * Yields CustomEvent chunks during execution and pushes the result to the results array.
458
545
  */
459
- async function* executeServerTool<TContext = unknown>(
546
+ export async function* executeServerTool<TContext = unknown>(
460
547
  toolCall: ToolCall,
461
548
  tool: AnyTool,
462
549
  toolName: string,
@@ -475,7 +562,14 @@ async function* executeServerTool<TContext = unknown>(
475
562
  let result = yield* executeWithEventPolling(executionPromise, pendingEvents)
476
563
  const duration = Date.now() - startTime
477
564
 
478
- // Flush remaining events
565
+ // MCP Apps: if this tool links a ui:// resource, eagerly read it and queue
566
+ // a `ui-resource` CUSTOM event. The MCP source stays live until the run
567
+ // drains (MCPManager's `connection:'close'` policy disposes on completion),
568
+ // so `readResource` is callable here. Fail-soft: a read error warns and
569
+ // emits nothing — the text result still flows.
570
+ await emitUiResourceIfLinked(tool, context)
571
+
572
+ // Flush remaining events (including any queued ui-resource event)
479
573
  let pendingEvent: CustomEvent | undefined
480
574
  while ((pendingEvent = pendingEvents.shift()) !== undefined) {
481
575
  yield pendingEvent
package/src/client.ts CHANGED
@@ -114,6 +114,7 @@ export type {
114
114
  ToolCallPart,
115
115
  ToolResultPart,
116
116
  UIMessage,
117
+ UIResourcePart,
117
118
  VideoPart,
118
119
  InferSchemaType,
119
120
  } from './types'
package/src/types.ts CHANGED
@@ -416,6 +416,23 @@ export interface StructuredOutputPart<TData = unknown> {
416
416
  errorMessage?: string
417
417
  }
418
418
 
419
+ export interface UIResourcePart {
420
+ type: 'ui-resource'
421
+ /** The ui:// resource object in MCP-native shape — fed straight to the renderer. */
422
+ resource: { uri: string; mimeType: string; text?: string; blob?: string }
423
+ /** Pool prefix / config key — routes interactive calls to the right MCP server. */
424
+ serverId?: string
425
+ /** Links the widget to the originating tool call — correlates it with the
426
+ * sibling ToolCallPart/ToolResultPart in the same message. */
427
+ toolCallId: string
428
+ /** Server-native (unprefixed) MCP tool name whose UI this resource renders.
429
+ * Required by the renderer (`@mcp-ui/client`'s `AppRenderer` `toolName` prop). */
430
+ toolName: string
431
+ /** Reserved for future passthrough of the resource/tool `_meta.ui` (e.g. frame-size hints).
432
+ * Currently always `undefined` — nothing populates this field yet. */
433
+ meta?: Record<string, unknown>
434
+ }
435
+
419
436
  export type MessagePart<TData = unknown> =
420
437
  | TextPart
421
438
  | ImagePart
@@ -426,6 +443,7 @@ export type MessagePart<TData = unknown> =
426
443
  | ToolResultPart
427
444
  | ThinkingPart
428
445
  | StructuredOutputPart<TData>
446
+ | UIResourcePart
429
447
 
430
448
  /**
431
449
  * UIMessage - Domain-specific message format optimized for building chat UIs
@@ -1308,6 +1326,19 @@ export interface ToolInputAvailableEvent extends CustomEvent {
1308
1326
  }
1309
1327
  }
1310
1328
 
1329
+ /** Emitted when an MCP tool returns a ui:// resource (MCP Apps). Reconciled into
1330
+ * a UIResourcePart on the assistant UIMessage. Never enters model input. */
1331
+ export interface UIResourceEvent extends CustomEvent {
1332
+ name: 'ui-resource'
1333
+ value: {
1334
+ resource: UIResourcePart['resource']
1335
+ serverId?: string
1336
+ toolCallId: string
1337
+ toolName: string
1338
+ meta?: Record<string, unknown>
1339
+ }
1340
+ }
1341
+
1311
1342
  /**
1312
1343
  * Public type for streams returned by `chat({ outputSchema, stream: true })`.
1313
1344
  *