ai 7.0.161 → 7.0.163

package/src/transcribe/transcription-events.ts

@@ -1,4 +1,8 @@
-import type { JSONObject, SharedV4AudioFormat } from '@ai-sdk/provider';
+import type {
+  JSONObject,
+  SharedV4AudioFormat,
+  TranscriptionModelV4TimestampGranularity,
+} from '@ai-sdk/provider';
 import type { ProviderOptions } from '@ai-sdk/provider-utils';
 import type { ProviderMetadata } from '../types';
 import type { TranscriptionModelResponseMetadata } from '../types/transcription-model-response-metadata';
@@ -21,8 +25,9 @@ type TranscriptionStartEventFor<OPERATION_ID extends string> = {
   readonly modelId: string;
 
   /**
-   * Metadata about the input audio. The byte length is known immediately for
-   * non-streaming transcription and is reported on the end event for streams.
+   * Metadata about the input audio. Native URL transcription does not inspect
+   * the audio, so its byte length and media type are unknown. Streaming byte
+   * length is reported on the end event after consuming the stream.
    */
   readonly audio: {
     readonly byteLength: number | undefined;
@@ -47,7 +52,10 @@ type TranscriptionStartEventFor<OPERATION_ID extends string> = {
 };
 
 export type TranscriptionStartEvent =
-  TranscriptionStartEventFor<'ai.transcribe'>;
+  TranscriptionStartEventFor<'ai.transcribe'> & {
+    /** Requested shared timing granularities, when specified. */
+    readonly timestampGranularities?: Array<TranscriptionModelV4TimestampGranularity>;
+  };
 
 export type StreamTranscriptionStartEvent =
   TranscriptionStartEventFor<'ai.streamTranscribe'>;
@@ -72,14 +80,18 @@ type TranscriptionEndEventFor<OPERATION_ID extends string> = {
 
   /** Metadata about the input audio. */
   readonly audio: {
-    readonly byteLength: number;
-    readonly mediaType: string;
+    readonly byteLength: OPERATION_ID extends 'ai.transcribe'
+      ? number | undefined
+      : number;
+    readonly mediaType: OPERATION_ID extends 'ai.transcribe'
+      ? string | undefined
+      : string;
   };
 
   /** The complete transcript. */
   readonly text: string;
 
-  /** Transcript segments with timing information. */
+  /** Transcript segments with timing information, retaining provider behavior. */
   readonly segments: Array<{
     readonly text: string;
     readonly startSecond: number;
@@ -105,7 +117,15 @@ type TranscriptionEndEventFor<OPERATION_ID extends string> = {
   readonly response: TranscriptionModelResponseMetadata;
 };
 
-export type TranscriptionEndEvent = TranscriptionEndEventFor<'ai.transcribe'>;
+export type TranscriptionEndEvent =
+  TranscriptionEndEventFor<'ai.transcribe'> & {
+    /** Word timings returned by the provider, whether requested or returned by default. */
+    readonly words?: Array<{
+      readonly text: string;
+      readonly startSecond: number;
+      readonly endSecond: number;
+    }>;
+  };
 
 export type StreamTranscriptionEndEvent =
   TranscriptionEndEventFor<'ai.streamTranscribe'>;

package/src/types/language-model-request-metadata.ts

@@ -4,6 +4,9 @@ import type { ModelMessage } from '@ai-sdk/provider-utils';
  * Metadata for a language model request.
  */
 export type LanguageModelRequestMetadata = {
+  /** HTTP origin used for this request, without URL credentials, paths, queries or fragments. */
+  readonly url?: string;
+
   /**
    * The input messages that were sent to the model for this step.
    */

package/src/types/realtime-model.ts

@@ -4,9 +4,12 @@ import type {
   Experimental_RealtimeFactoryV4GetTokenResult as RealtimeFactoryV4GetTokenResult,
   Experimental_RealtimeModelV4 as RealtimeModelV4,
   Experimental_RealtimeModelV4ClientEvent as RealtimeModelV4ClientEvent,
+  Experimental_RealtimeModelV4ServerConnection as RealtimeModelV4ServerConnection,
   Experimental_RealtimeModelV4ServerEvent as RealtimeModelV4ServerEvent,
   Experimental_RealtimeModelV4SessionConfig as RealtimeModelV4SessionConfig,
+  Experimental_RealtimeModelV4SessionIntent as RealtimeModelV4SessionIntent,
   Experimental_RealtimeModelV4ToolDefinition as RealtimeModelV4ToolDefinition,
+  Experimental_RealtimeModelV4Usage as RealtimeModelV4Usage,
 } from '@ai-sdk/provider';
 
 export type RealtimeFactory = RealtimeFactoryV4;
@@ -19,8 +22,14 @@ export type RealtimeModel = RealtimeModelV4;
 
 export type RealtimeClientEvent = RealtimeModelV4ClientEvent;
 
+export type RealtimeServerConnection = RealtimeModelV4ServerConnection;
+
 export type RealtimeServerEvent = RealtimeModelV4ServerEvent;
 
 export type RealtimeSessionConfig = RealtimeModelV4SessionConfig;
 
+export type RealtimeSessionIntent = RealtimeModelV4SessionIntent;
+
 export type RealtimeToolDefinition = RealtimeModelV4ToolDefinition;
+
+export type RealtimeUsage = RealtimeModelV4Usage;

package/src/ui/convert-to-model-messages.ts

@@ -148,34 +148,8 @@ export async function convertToModelMessages<UI_MESSAGE extends UIMessage>(
           role: 'user',
           content: message.parts
             .map((part): TextPart | FilePart | undefined => {
-              // Process text parts
-              if (isTextUIPart(part)) {
-                return {
-                  type: 'text' as const,
-                  text: part.text,
-                  ...(part.providerMetadata != null
-                    ? { providerOptions: part.providerMetadata }
-                    : {}),
-                };
-              }
-
-              // Process file parts
-              if (isFileUIPart(part)) {
-                return {
-                  type: 'file' as const,
-                  mediaType: part.mediaType,
-                  filename: part.filename,
-                  data:
-                    part.providerReference != null
-                      ? {
-                          type: 'reference' as const,
-                          reference: part.providerReference,
-                        }
-                      : { type: 'url' as const, url: new URL(part.url) },
-                  ...(part.providerMetadata != null
-                    ? { providerOptions: part.providerMetadata }
-                    : {}),
-                };
+              if (isTextUIPart(part) || isFileUIPart(part)) {
+                return convertUserContentUIPart(part);
               }
 
               // Process data parts with converter if provided
@@ -481,6 +455,13 @@ export async function convertToModelMessages<UI_MESSAGE extends UIMessage>(
               block.push(part as (typeof block)[number]);
             } else if (part.type === 'step-start') {
               await processBlock();
+            } else if (part.type === 'steering-message') {
+              // steering messages were added to the run between two steps:
+              await processBlock();
+              modelMessages.push({
+                role: 'user',
+                content: part.parts.map(convertUserContentUIPart),
+              });
             }
           }
 
@@ -504,3 +485,30 @@ export async function convertToModelMessages<UI_MESSAGE extends UIMessage>(
 
   return modelMessages;
 }
+
+function convertUserContentUIPart(
+  part: TextUIPart | FileUIPart,
+): TextPart | FilePart {
+  if (part.type === 'text') {
+    return {
+      type: 'text',
+      text: part.text,
+      ...(part.providerMetadata != null
+        ? { providerOptions: part.providerMetadata }
+        : {}),
+    };
+  }
+
+  return {
+    type: 'file',
+    mediaType: part.mediaType,
+    filename: part.filename,
+    data:
+      part.providerReference != null
+        ? { type: 'reference', reference: part.providerReference }
+        : { type: 'url', url: new URL(part.url) },
+    ...(part.providerMetadata != null
+      ? { providerOptions: part.providerMetadata }
+      : {}),
+  };
+}

package/src/ui/index.ts

@@ -69,6 +69,7 @@ export {
   type ReasoningUIPart,
   type SourceDocumentUIPart,
   type SourceUrlUIPart,
+  type SteeringMessageUIPart as Experimental_SteeringMessageUIPart,
   type StepStartUIPart,
   type TextUIPart,
   type ToolOutputErrorUIPart,

package/src/ui/process-ui-message-stream.ts

@@ -961,6 +961,16 @@ export function processUIMessageStream<UI_MESSAGE extends UIMessage>({
               break;
             }
 
+            case 'steering-message': {
+              state.message.parts.push({
+                type: 'steering-message',
+                id: chunk.id,
+                parts: chunk.parts,
+              });
+              write();
+              break;
+            }
+
             case 'start-step': {
               // add a step boundary part to the message
               state.message.parts.push({ type: 'step-start' });

package/src/ui/ui-messages.ts

@@ -89,6 +89,7 @@ export type UIMessagePart<
   | FileUIPart
   | ReasoningFileUIPart
   | DataUIPart<DATA_TYPES>
+  | SteeringMessageUIPart
   | StepStartUIPart;
 
 /**
@@ -252,6 +253,25 @@ export type ReasoningFileUIPart = {
   providerMetadata?: ProviderMetadata;
 };
 
+/**
+ * Experimental. A user message that was added to the run with steering while
+ * the assistant message was generated. It is placed before the output of the
+ * step that received it.
+ */
+export type SteeringMessageUIPart = {
+  type: 'steering-message';
+
+  /**
+   * The id of the steering message.
+   */
+  id: string;
+
+  /**
+   * The content of the user message.
+   */
+  parts: Array<TextUIPart | FileUIPart>;
+};
+
 /**
  * A step boundary part of a message.
  */

package/src/ui/validate-ui-messages.ts

@@ -141,6 +141,27 @@ const uiMessagesSchema = lazySchema(() => {
                   url: z.string(),
                   providerMetadata: providerMetadataSchema.optional(),
                 }),
+                z.object({
+                  type: z.literal('steering-message'),
+                  id: z.string(),
+                  parts: z.array(
+                    z.union([
+                      z.object({
+                        type: z.literal('text'),
+                        text: z.string(),
+                        providerMetadata: providerMetadataSchema.optional(),
+                      }),
+                      z.object({
+                        type: z.literal('file'),
+                        mediaType: z.string(),
+                        filename: z.string().optional(),
+                        url: z.string(),
+                        providerReference: providerReferenceSchema.optional(),
+                        providerMetadata: providerMetadataSchema.optional(),
+                      }),
+                    ]),
+                  ),
+                }),
                 z.object({
                   type: z.literal('step-start'),
                 }),

package/src/ui-message-stream/to-ui-message-chunk.ts

@@ -1,9 +1,17 @@
-import type { ToolSet } from '@ai-sdk/provider-utils';
+import {
+  convertUint8ArrayToBase64,
+  type ToolSet,
+  type UserModelMessage,
+} from '@ai-sdk/provider-utils';
 import type { TextStreamPart } from '../generate-text/stream-text-result';
 import { getToolCallInputSchemaInput } from '../generate-text/tool-call';
+import { convertDataContentToBase64String } from '../prompt/data-content';
+import { convertToLanguageModelV4FilePart } from '../prompt/file-part-data';
 import type {
+  FileUIPart,
   InferUIMessageData,
   InferUIMessageMetadata,
+  TextUIPart,
   UIMessage,
 } from '../ui/ui-messages';
 import { isDeepEqualData } from '../util/is-deep-equal-data';
@@ -352,6 +360,14 @@ export function toUIMessageChunk<
       };
     }
 
+    case 'steering-message': {
+      return {
+        type: 'steering-message',
+        id: part.id,
+        parts: convertUserMessageToUIParts(part.message),
+      };
+    }
+
     case 'start-step': {
       return { type: 'start-step' };
     }
@@ -395,6 +411,7 @@ export function toUIMessageChunk<
     }
 
     case 'tool-input-end':
+    case 'ping':
     case 'raw': {
       return undefined;
     }
@@ -405,3 +422,57 @@ export function toUIMessageChunk<
     }
   }
 }
+
+function convertUserMessageToUIParts(
+  message: UserModelMessage,
+): Array<TextUIPart | FileUIPart> {
+  if (typeof message.content === 'string') {
+    return [{ type: 'text', text: message.content }];
+  }
+
+  return message.content.map((part): TextUIPart | FileUIPart => {
+    const providerMetadata =
+      part.providerOptions != null
+        ? { providerMetadata: part.providerOptions }
+        : {};
+
+    if (part.type === 'text') {
+      return { type: 'text', text: part.text, ...providerMetadata };
+    }
+
+    const { data, mediaType: dataUrlMediaType } =
+      convertToLanguageModelV4FilePart(
+        part.type === 'image' ? part.image : part.data,
+      );
+    const mediaType = dataUrlMediaType ?? part.mediaType ?? 'image';
+    const filename = part.type === 'file' ? part.filename : undefined;
+    const file = {
+      type: 'file' as const,
+      mediaType,
+      ...(filename != null ? { filename } : {}),
+      ...providerMetadata,
+    };
+
+    switch (data.type) {
+      case 'url': {
+        return { ...file, url: data.url.toString() };
+      }
+      case 'data': {
+        return {
+          ...file,
+          url: `data:${mediaType};base64,${convertDataContentToBase64String(data.data)}`,
+        };
+      }
+      case 'text': {
+        return {
+          ...file,
+          url: `data:${mediaType};base64,${convertUint8ArrayToBase64(new TextEncoder().encode(data.text))}`,
+        };
+      }
+      case 'reference': {
+        // the provider reference takes precedence over the url in model messages:
+        return { ...file, url: '', providerReference: data.reference };
+      }
+    }
+  });
+}

package/src/ui-message-stream/ui-message-chunks.ts

@@ -7,8 +7,10 @@ import {
 import { jsonValueSchema } from '../types/json-value';
 import type { FinishReason } from '../types/language-model';
 import type {
+  FileUIPart,
   InferUIMessageData,
   InferUIMessageMetadata,
+  TextUIPart,
   UIDataTypes,
   UIMessage,
 } from '../ui/ui-messages';
@@ -173,6 +175,27 @@ export const uiMessageChunkSchema = lazySchema(() =>
         mediaType: z.string(),
         providerMetadata: providerMetadataSchema.optional(),
       }),
+      z.looseObject({
+        type: z.literal('steering-message'),
+        id: z.string(),
+        parts: z.array(
+          z.union([
+            z.looseObject({
+              type: z.literal('text'),
+              text: z.string(),
+              providerMetadata: providerMetadataSchema.optional(),
+            }),
+            z.looseObject({
+              type: z.literal('file'),
+              mediaType: z.string(),
+              url: z.string(),
+              filename: z.string().optional(),
+              providerReference: z.record(z.string(), z.string()).optional(),
+              providerMetadata: providerMetadataSchema.optional(),
+            }),
+          ]),
+        ),
+      }),
       z.looseObject({
         type: z.custom<`data-${string}`>(
           (value): value is `data-${string}` =>
@@ -388,6 +411,14 @@ export type UIMessageChunk<
       providerMetadata?: ProviderMetadata;
     }
   | DataUIMessageChunk<DATA_TYPES>
+  | {
+      /**
+       * Experimental. A user message that was added to the run with steering.
+       */
+      type: 'steering-message';
+      id: string;
+      parts: Array<TextUIPart | FileUIPart>;
+    }
   | {
       type: 'start-step';
     }