ai 7.0.161 → 7.0.163

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/CHANGELOG.md +79 -0
  2. package/dist/index.d.ts +210 -41
  3. package/dist/index.d.ts.map +1 -1
  4. package/dist/index.js +551 -114
  5. package/dist/index.js.map +1 -1
  6. package/dist/internal/index.d.ts +87 -10
  7. package/dist/internal/index.d.ts.map +1 -1
  8. package/dist/internal/index.js +221 -26
  9. package/dist/internal/index.js.map +1 -1
  10. package/dist/test/index.d.ts +3 -1
  11. package/dist/test/index.d.ts.map +1 -1
  12. package/dist/test/index.js +2 -1
  13. package/dist/test/index.js.map +1 -1
  14. package/docs/02-foundations/02-providers-and-models.mdx +2 -0
  15. package/docs/03-agents/04-loop-control.mdx +34 -0
  16. package/docs/03-agents/06-memory.mdx +42 -0
  17. package/docs/03-agents/06-tool-approvals.mdx +2 -0
  18. package/docs/03-ai-sdk-core/05-generating-text.mdx +1 -0
  19. package/docs/03-ai-sdk-core/36-realtime.mdx +74 -1
  20. package/docs/03-ai-sdk-core/36-transcription.mdx +51 -4
  21. package/docs/03-ai-sdk-core/40-middleware.mdx +7 -2
  22. package/docs/03-ai-sdk-core/60-telemetry.mdx +85 -4
  23. package/docs/04-ai-sdk-ui/50-stream-protocol.mdx +13 -0
  24. package/docs/06-advanced/04-caching.mdx +13 -4
  25. package/docs/07-reference/01-ai-sdk-core/01-generate-text.mdx +33 -28
  26. package/docs/07-reference/01-ai-sdk-core/02-stream-text.mdx +58 -1
  27. package/docs/07-reference/01-ai-sdk-core/11-transcribe.mdx +17 -2
  28. package/docs/07-reference/01-ai-sdk-core/13-generate-video.mdx +8 -1
  29. package/docs/07-reference/01-ai-sdk-core/31-ui-message.mdx +23 -0
  30. package/docs/07-reference/02-ai-sdk-ui/05-use-realtime.mdx +37 -8
  31. package/docs/07-reference/04-ai-sdk-workflow/01-workflow-agent.mdx +1 -49
  32. package/docs/08-migration-guides/23-migration-guide-7-0.mdx +36 -34
  33. package/package.json +12 -12
  34. package/src/generate-object/generate-object.ts +2 -1
  35. package/src/generate-object/stream-object.ts +7 -3
  36. package/src/generate-text/generate-text.ts +23 -3
  37. package/src/generate-text/is-output-chunk.ts +1 -0
  38. package/src/generate-text/language-model-events.ts +3 -0
  39. package/src/generate-text/output.ts +90 -37
  40. package/src/generate-text/smooth-stream.ts +5 -0
  41. package/src/generate-text/steering.ts +5 -4
  42. package/src/generate-text/stream-language-model-call.ts +16 -1
  43. package/src/generate-text/stream-text-result.ts +29 -1
  44. package/src/generate-text/stream-text.ts +25 -1
  45. package/src/generate-text/to-response-messages.ts +6 -2
  46. package/src/generate-text/validate-tool-approvals.ts +21 -3
  47. package/src/generate-video/generate-video-result.ts +11 -2
  48. package/src/generate-video/generate-video.ts +26 -2
  49. package/src/model/as-transcription-model-v4.ts +25 -0
  50. package/src/prompt/convert-to-language-model-prompt.ts +86 -14
  51. package/src/realtime/index.ts +3 -0
  52. package/src/realtime/realtime-attempt.ts +1 -0
  53. package/src/realtime/realtime-session.ts +142 -21
  54. package/src/telemetry/create-telemetry-dispatcher.ts +8 -1
  55. package/src/telemetry/filter-telemetry-event.ts +165 -0
  56. package/src/telemetry/get-request-origin.ts +25 -0
  57. package/src/telemetry/index.ts +6 -1
  58. package/src/telemetry/telemetry.ts +62 -0
  59. package/src/telemetry/tracing-channel-publisher.ts +19 -4
  60. package/src/test/mock-transcription-model-v4.ts +4 -0
  61. package/src/transcribe/transcribe-result.ts +20 -0
  62. package/src/transcribe/transcribe.ts +107 -20
  63. package/src/transcribe/transcription-events.ts +28 -8
  64. package/src/types/language-model-request-metadata.ts +3 -0
  65. package/src/types/realtime-model.ts +9 -0
  66. package/src/ui/convert-to-model-messages.ts +36 -28
  67. package/src/ui/index.ts +1 -0
  68. package/src/ui/process-ui-message-stream.ts +10 -0
  69. package/src/ui/ui-messages.ts +20 -0
  70. package/src/ui/validate-ui-messages.ts +21 -0
  71. package/src/ui-message-stream/to-ui-message-chunk.ts +72 -1
  72. package/src/ui-message-stream/ui-message-chunks.ts +31 -0
@@ -110,7 +110,7 @@ function asAsyncIterableStream(stream) {
110
110
  }
111
111
  //#endregion
112
112
  //#region src/version.ts
113
- const VERSION = "7.0.161";
113
+ const VERSION = "7.0.163";
114
114
  //#endregion
115
115
  //#region src/util/download/download.ts
116
116
  /**
@@ -736,12 +736,18 @@ async function downloadAssets(messages, download, supportedUrls) {
736
736
  if (message.role === "tool") for (const part of message.content) {
737
737
  if (part.type !== "tool-result") continue;
738
738
  if (part.output.type !== "content") continue;
739
- for (const contentPart of part.output.value) if (contentPart.type === "file") downloadableFiles.push(contentPart);
739
+ for (const contentPart of part.output.value) {
740
+ const filePart = getDownloadableFilePart(contentPart);
741
+ if (filePart != null) downloadableFiles.push(filePart);
742
+ }
740
743
  }
741
744
  if (message.role === "assistant" && Array.isArray(message.content)) for (const part of message.content) {
742
745
  if (part.type !== "tool-result") continue;
743
746
  if (part.output.type !== "content") continue;
744
- for (const contentPart of part.output.value) if (contentPart.type === "file") downloadableFiles.push(contentPart);
747
+ for (const contentPart of part.output.value) {
748
+ const filePart = getDownloadableFilePart(contentPart);
749
+ if (filePart != null) downloadableFiles.push(filePart);
750
+ }
745
751
  }
746
752
  }
747
753
  const plannedDownloads = downloadableFiles.map((part) => {
@@ -766,6 +772,57 @@ async function downloadAssets(messages, download, supportedUrls) {
766
772
  }]).filter((file) => file != null));
767
773
  }
768
774
  /**
775
+ * Resolves the tool result content parts that point at a remote file to the
776
+ * tagged file shape, including the deprecated `file-url` and `image-url`
777
+ * types, so that they are downloaded and inlined exactly like their `file`
778
+ * replacement. Returns `undefined` for content that is already inline or
779
+ * provider-referenced.
780
+ */
781
+ function getDownloadableFilePart(contentPart) {
782
+ switch (contentPart.type) {
783
+ case "file": return contentPart;
784
+ case "file-url": return {
785
+ type: "file",
786
+ data: {
787
+ type: "url",
788
+ url: new URL(contentPart.url)
789
+ },
790
+ mediaType: contentPart.mediaType ?? getMediaTypeFromUrl(contentPart.url),
791
+ providerOptions: contentPart.providerOptions
792
+ };
793
+ case "image-url": return {
794
+ type: "file",
795
+ data: {
796
+ type: "url",
797
+ url: new URL(contentPart.url)
798
+ },
799
+ mediaType: "image",
800
+ providerOptions: contentPart.providerOptions
801
+ };
802
+ default: return;
803
+ }
804
+ }
805
+ /**
806
+ * Resolves file data and media type for a remote tool result content part: the downloaded
807
+ * bytes when the asset was fetched, otherwise the URL reference. `originalUrl`
808
+ * is preserved when the URL string does not round-trip through `URL`.
809
+ */
810
+ function resolveDownloadedFile({ url, originalUrl, downloadedAssets, mediaType }) {
811
+ const convertedPart = convertPartToLanguageModelPart({
812
+ type: "file",
813
+ data: {
814
+ type: "url",
815
+ url,
816
+ ...url.toString() !== originalUrl ? { originalUrl } : {}
817
+ },
818
+ mediaType
819
+ }, downloadedAssets);
820
+ return {
821
+ data: convertedPart.data,
822
+ mediaType: convertedPart.mediaType
823
+ };
824
+ }
825
+ /**
769
826
  * Convert part of a user message to a LanguageModelV4Part.
770
827
  *
771
828
  * @param part - The part to convert.
@@ -850,12 +907,12 @@ function mapToolResultOutput({ output, provider, warnings = [], downloadedAssets
850
907
  });
851
908
  return {
852
909
  type: "file",
853
- data: {
854
- type: "url",
910
+ ...resolveDownloadedFile({
855
911
  url,
856
- ...url.toString() !== item.url ? { originalUrl: item.url } : {}
857
- },
858
- mediaType,
912
+ originalUrl: item.url,
913
+ downloadedAssets,
914
+ mediaType
915
+ }),
859
916
  providerOptions: item.providerOptions
860
917
  };
861
918
  }
@@ -916,12 +973,12 @@ function mapToolResultOutput({ output, provider, warnings = [], downloadedAssets
916
973
  });
917
974
  return {
918
975
  type: "file",
919
- data: {
920
- type: "url",
976
+ ...resolveDownloadedFile({
921
977
  url,
922
- ...url.toString() !== item.url ? { originalUrl: item.url } : {}
923
- },
924
- mediaType: "image",
978
+ originalUrl: item.url,
979
+ downloadedAssets,
980
+ mediaType: "image"
981
+ }),
925
982
  providerOptions: item.providerOptions
926
983
  };
927
984
  }
@@ -1784,6 +1841,18 @@ function asTranscriptionModelV4(model) {
1784
1841
  const v3Model = model.specificationVersion === "v2" ? asTranscriptionModelV3(model) : model;
1785
1842
  return new Proxy(v3Model, { get(target, prop) {
1786
1843
  if (prop === "specificationVersion") return "v4";
1844
+ if (prop === "doGenerate") return async ({ timestampGranularities, ...options }) => {
1845
+ const result = await target.doGenerate.call(model, options);
1846
+ if (!timestampGranularities?.length) return result;
1847
+ return {
1848
+ ...result,
1849
+ warnings: [...result.warnings, {
1850
+ type: "unsupported",
1851
+ feature: "timestampGranularities",
1852
+ details: `Shared timestamp granularities are not supported by ${model.specificationVersion} transcription models.`
1853
+ }]
1854
+ };
1855
+ };
1787
1856
  return target[prop];
1788
1857
  } });
1789
1858
  }
@@ -1936,7 +2005,7 @@ function loadBuiltinModule(id) {
1936
2005
  * if tracing itself throws, and prevents falling back by calling `execute` a
1937
2006
  * second time.
1938
2007
  */
1939
- async function runWithTracingChannelSpan(message, execute) {
2008
+ async function runWithTracingChannelSpan(message, execute, recordResult = true) {
1940
2009
  const tracingChannel = (await loadDiagnosticsChannel())?.tracingChannel?.(AI_SDK_TELEMETRY_TRACING_CHANNEL);
1941
2010
  if (tracingChannel == null || tracingChannel.hasSubscribers === false) return await execute();
1942
2011
  let executePromise;
@@ -1959,10 +2028,13 @@ async function runWithTracingChannelSpan(message, execute) {
1959
2028
  hasExecutionError = true;
1960
2029
  throw error;
1961
2030
  });
1962
- return executePromise;
2031
+ return recordResult ? executePromise : executePromise.then(() => void 0, () => {
2032
+ throw new Error("AI SDK operation failed");
2033
+ });
1963
2034
  };
1964
2035
  try {
1965
- return await tracingChannel.tracePromise(tracedExecute, message);
2036
+ await tracingChannel.tracePromise(tracedExecute, message);
2037
+ return executionResult;
1966
2038
  } catch {
1967
2039
  if (hasExecutionError) throw executionError;
1968
2040
  if (hasExecutionResult) return executionResult;
@@ -1974,7 +2046,7 @@ async function runWithTracingChannelSpan(message, execute) {
1974
2046
  * Opens a long-lived tracing-channel span context and returns a runner that can
1975
2047
  * re-enter that context later without changing stream setup timing.
1976
2048
  */
1977
- function openTelemetryChannelSpanContext({ message, completion }) {
2049
+ function openTelemetryChannelSpanContext({ message, completion, recordResult = true }) {
1978
2050
  if (!isNodeRuntime()) {
1979
2051
  Promise.resolve(completion).catch(() => {});
1980
2052
  return;
@@ -1998,10 +2070,10 @@ function openTelemetryChannelSpanContext({ message, completion }) {
1998
2070
  if (asyncEndPublished) return;
1999
2071
  asyncEndPublished = true;
2000
2072
  if (error !== void 0) {
2001
- context.error = error;
2073
+ context.error = recordResult ? error : /* @__PURE__ */ new Error("AI SDK operation failed");
2002
2074
  safePublish(() => tracingChannel.error.publish(context));
2003
2075
  }
2004
- if (result !== void 0) context.result = result;
2076
+ if (recordResult && result !== void 0) context.result = result;
2005
2077
  safePublish(() => tracingChannel.asyncEnd.publish(context));
2006
2078
  };
2007
2079
  safePublish(() => {
@@ -2019,6 +2091,116 @@ function getGlobalTelemetryIntegrations() {
2019
2091
  return globalThis.AI_SDK_TELEMETRY_INTEGRATIONS ?? [];
2020
2092
  }
2021
2093
  //#endregion
2094
+ //#region src/telemetry/filter-telemetry-event.ts
2095
+ /**
2096
+ * Filters SDK-owned content fields on a copy. Only descend into SDK event
2097
+ * containers; user context has its own inclusion policy.
2098
+ */
2099
+ function filterTelemetryEvent(event, telemetry) {
2100
+ const hideInputs = telemetry.recordInputs === false;
2101
+ const hideOutputs = telemetry.recordOutputs === false;
2102
+ if (!hideInputs && !hideOutputs || event == null || typeof event !== "object") return event;
2103
+ const filtered = Object.assign(Object.create(Object.getPrototypeOf(event)), event);
2104
+ const replace = (fields) => {
2105
+ for (const [key, value] of Object.entries(fields)) if (Object.hasOwn(event, key)) filtered[key] = value;
2106
+ };
2107
+ const isSpeech = filtered.operationId === "ai.generateSpeech";
2108
+ if (hideInputs) {
2109
+ replace({
2110
+ instructions: void 0,
2111
+ system: void 0,
2112
+ prompt: void 0,
2113
+ messages: [],
2114
+ promptMessages: [],
2115
+ experimental_steeringMessages: [],
2116
+ tools: void 0,
2117
+ stepTools: void 0,
2118
+ schema: void 0,
2119
+ schemaName: void 0,
2120
+ schemaDescription: void 0,
2121
+ ...typeof filtered.output === "object" ? { output: void 0 } : {},
2122
+ providerOptions: {},
2123
+ value: "",
2124
+ values: [],
2125
+ query: "",
2126
+ documents: [],
2127
+ state: Array.isArray(filtered.state) ? [] : typeof filtered.state === "string" ? "" : {},
2128
+ questions: {},
2129
+ ...isSpeech ? { text: "" } : {}
2130
+ });
2131
+ if (filtered.toolCall != null) filtered.toolCall = {
2132
+ ...filtered.toolCall,
2133
+ input: void 0
2134
+ };
2135
+ if (filtered.toolOutput != null) filtered.toolOutput = {
2136
+ ...filtered.toolOutput,
2137
+ input: void 0
2138
+ };
2139
+ if (Array.isArray(filtered.ranking)) filtered.ranking = filtered.ranking.map((entry) => "document" in entry ? {
2140
+ ...entry,
2141
+ document: typeof entry.document === "string" ? "" : {}
2142
+ } : entry);
2143
+ }
2144
+ if (hideOutputs) {
2145
+ replace({
2146
+ content: [],
2147
+ ...!isSpeech ? { text: "" } : {},
2148
+ reasoning: Array.isArray(filtered.reasoning) ? [] : void 0,
2149
+ reasoningText: void 0,
2150
+ files: [],
2151
+ sources: [],
2152
+ toolCalls: [],
2153
+ staticToolCalls: [],
2154
+ dynamicToolCalls: [],
2155
+ toolResults: [],
2156
+ staticToolResults: [],
2157
+ dynamicToolResults: [],
2158
+ responseMessages: [],
2159
+ object: void 0,
2160
+ objectText: "",
2161
+ embedding: [],
2162
+ embeddings: [],
2163
+ ranking: [],
2164
+ answers: {},
2165
+ segments: []
2166
+ });
2167
+ if (filtered.toolOutput != null) filtered.toolOutput = {
2168
+ ...filtered.toolOutput,
2169
+ output: void 0,
2170
+ error: void 0
2171
+ };
2172
+ }
2173
+ replace({
2174
+ providerMetadata: void 0,
2175
+ error: filtered.error == null ? filtered.error : /* @__PURE__ */ new Error("AI SDK operation failed")
2176
+ });
2177
+ for (const key of ["toolCall", "toolOutput"]) {
2178
+ const toolEvent = filtered[key];
2179
+ if (toolEvent != null && typeof toolEvent === "object") filtered[key] = {
2180
+ ...toolEvent,
2181
+ providerMetadata: void 0,
2182
+ ..."error" in toolEvent ? { error: toolEvent.error == null ? toolEvent.error : /* @__PURE__ */ new Error("AI SDK operation failed") } : {}
2183
+ };
2184
+ }
2185
+ for (const key of ["request", "response"]) {
2186
+ const filterMetadata = (metadata) => {
2187
+ if (metadata == null || typeof metadata !== "object") return metadata;
2188
+ const filteredMetadata = { ...metadata };
2189
+ delete filteredMetadata.body;
2190
+ if (key === "request" ? hideInputs : hideOutputs) {
2191
+ if ("messages" in metadata) filteredMetadata.messages = [];
2192
+ if ("instructions" in metadata) filteredMetadata.instructions = void 0;
2193
+ }
2194
+ return filteredMetadata;
2195
+ };
2196
+ const metadata = filtered[key];
2197
+ if (metadata != null) filtered[key] = Array.isArray(metadata) ? metadata.map(filterMetadata) : filterMetadata(metadata);
2198
+ }
2199
+ if (Array.isArray(filtered.steps)) filtered.steps = filtered.steps.map((step) => filterTelemetryEvent(step, telemetry));
2200
+ if (Object.hasOwn(event, "finalStep")) filtered.finalStep = filterTelemetryEvent(filtered.finalStep, telemetry);
2201
+ return filtered;
2202
+ }
2203
+ //#endregion
2022
2204
  //#region src/telemetry/create-telemetry-dispatcher.ts
2023
2205
  function augmentEvent(event, telemetry, filterContext = false) {
2024
2206
  const augmentedEvent = Object.assign(Object.create(Object.getPrototypeOf(event)), event, {
@@ -2042,7 +2224,7 @@ function augmentEvent(event, telemetry, filterContext = false) {
2042
2224
  includeToolsContext: telemetry.includeToolsContext
2043
2225
  });
2044
2226
  }
2045
- return augmentedEvent;
2227
+ return filterTelemetryEvent(augmentedEvent, telemetry);
2046
2228
  }
2047
2229
  /**
2048
2230
  * Creates a telemetry dispatcher that sends telemetry events
@@ -2068,6 +2250,7 @@ function createTelemetryDispatcher({ telemetry }) {
2068
2250
  includeRuntimeContext: telemetry?.includeRuntimeContext,
2069
2251
  includeToolsContext: telemetry?.includeToolsContext
2070
2252
  };
2253
+ const recordResult = telemetry?.recordInputs !== false && telemetry?.recordOutputs !== false;
2071
2254
  const mergeTelemetryCallback = (key, deprecatedKey) => {
2072
2255
  const integrationCallbacks = integrations.map((integration) => {
2073
2256
  const callback = integration[key] ?? (deprecatedKey == null ? void 0 : integration[deprecatedKey]);
@@ -2087,13 +2270,14 @@ function createTelemetryDispatcher({ telemetry }) {
2087
2270
  runInTracingChannelSpan: async ({ type, event, execute }) => await runWithTracingChannelSpan({
2088
2271
  type,
2089
2272
  event: augmentEvent(event, telemetryMetadata, true)
2090
- }, execute),
2273
+ }, execute, recordResult),
2091
2274
  startTracingChannelContext: ({ type, event, completion }) => openTelemetryChannelSpanContext({
2092
2275
  message: {
2093
2276
  type,
2094
2277
  event: augmentEvent(event, telemetryMetadata, true)
2095
2278
  },
2096
- completion
2279
+ completion,
2280
+ recordResult
2097
2281
  }),
2098
2282
  onStart: mergeTelemetryCallback("onStart"),
2099
2283
  onStepStart: mergeTelemetryCallback("onStepStart"),
@@ -2135,7 +2319,7 @@ function createTelemetryDispatcher({ telemetry }) {
2135
2319
  return await runWithTracingChannelSpan({
2136
2320
  type: "languageModelCall",
2137
2321
  event: augmentedEvent
2138
- }, wrappedExecute);
2322
+ }, wrappedExecute, recordResult);
2139
2323
  },
2140
2324
  /**
2141
2325
  * Composes all `executeTool` wrappers around the original tool execution.
@@ -2626,6 +2810,7 @@ const isOutputChunkType = {
2626
2810
  "model-call-response-metadata": false,
2627
2811
  "model-call-end": false,
2628
2812
  error: false,
2813
+ ping: false,
2629
2814
  raw: false
2630
2815
  };
2631
2816
  function isOutputChunk(chunk) {
@@ -2955,9 +3140,19 @@ async function validateApprovedToolApprovals({ approvedToolApprovals, tools, too
2955
3140
  const approved = [];
2956
3141
  const denied = [];
2957
3142
  const invalid = [];
2958
- for (const approval of approvedToolApprovals) {
3143
+ for (const collectedApproval of approvedToolApprovals) {
3144
+ const tool = getOwn(tools, collectedApproval.toolCall.toolName);
3145
+ const { toolMetadata: _toolMetadata, dynamic: _dynamic, ...toolCallFromHistory } = collectedApproval.toolCall;
3146
+ const approval = {
3147
+ ...collectedApproval,
3148
+ toolCall: {
3149
+ ...toolCallFromHistory,
3150
+ ...tool?.metadata != null ? { toolMetadata: tool.metadata } : {},
3151
+ ...tool?.type === "dynamic" ? { dynamic: true } : {},
3152
+ title: tool?.title
3153
+ }
3154
+ };
2959
3155
  const { approvalRequest, toolCall } = approval;
2960
- const tool = getOwn(tools, toolCall.toolName);
2961
3156
  if (toolApprovalSecret != null) {
2962
3157
  if (approvalRequest.signature == null) throw new InvalidToolApprovalSignatureError({
2963
3158
  approvalId: approvalRequest.approvalId,
@@ -3045,7 +3240,7 @@ async function toResponseMessages({ content: inputContent, tools }) {
3045
3240
  for (const part of inputContent) {
3046
3241
  if (part.type === "source") continue;
3047
3242
  if ((part.type === "tool-result" || part.type === "tool-error") && !part.providerExecuted) continue;
3048
- if (part.type === "text" && part.text.length === 0) continue;
3243
+ if (part.type === "text" && part.text.length === 0 && part.providerMetadata == null) continue;
3049
3244
  switch (part.type) {
3050
3245
  case "text":
3051
3246
  content.push({