@fondation-io/ai 7.0.0-beta.45

package/src/generate-object/validate-object-generation-input.ts

@@ -0,0 +1,144 @@
+import { FlexibleSchema } from '@ai-sdk/provider-utils';
+import { InvalidArgumentError } from '../error/invalid-argument-error';
+
+export function validateObjectGenerationInput({
+  output,
+  schema,
+  schemaName,
+  schemaDescription,
+  enumValues,
+}: {
+  output?: 'object' | 'array' | 'enum' | 'no-schema';
+  schema?: FlexibleSchema<unknown>;
+  schemaName?: string;
+  schemaDescription?: string;
+  enumValues?: Array<unknown>;
+}) {
+  if (
+    output != null &&
+    output !== 'object' &&
+    output !== 'array' &&
+    output !== 'enum' &&
+    output !== 'no-schema'
+  ) {
+    throw new InvalidArgumentError({
+      parameter: 'output',
+      value: output,
+      message: 'Invalid output type.',
+    });
+  }
+
+  if (output === 'no-schema') {
+    if (schema != null) {
+      throw new InvalidArgumentError({
+        parameter: 'schema',
+        value: schema,
+        message: 'Schema is not supported for no-schema output.',
+      });
+    }
+
+    if (schemaDescription != null) {
+      throw new InvalidArgumentError({
+        parameter: 'schemaDescription',
+        value: schemaDescription,
+        message: 'Schema description is not supported for no-schema output.',
+      });
+    }
+
+    if (schemaName != null) {
+      throw new InvalidArgumentError({
+        parameter: 'schemaName',
+        value: schemaName,
+        message: 'Schema name is not supported for no-schema output.',
+      });
+    }
+
+    if (enumValues != null) {
+      throw new InvalidArgumentError({
+        parameter: 'enumValues',
+        value: enumValues,
+        message: 'Enum values are not supported for no-schema output.',
+      });
+    }
+  }
+
+  if (output === 'object') {
+    if (schema == null) {
+      throw new InvalidArgumentError({
+        parameter: 'schema',
+        value: schema,
+        message: 'Schema is required for object output.',
+      });
+    }
+
+    if (enumValues != null) {
+      throw new InvalidArgumentError({
+        parameter: 'enumValues',
+        value: enumValues,
+        message: 'Enum values are not supported for object output.',
+      });
+    }
+  }
+
+  if (output === 'array') {
+    if (schema == null) {
+      throw new InvalidArgumentError({
+        parameter: 'schema',
+        value: schema,
+        message: 'Element schema is required for array output.',
+      });
+    }
+
+    if (enumValues != null) {
+      throw new InvalidArgumentError({
+        parameter: 'enumValues',
+        value: enumValues,
+        message: 'Enum values are not supported for array output.',
+      });
+    }
+  }
+
+  if (output === 'enum') {
+    if (schema != null) {
+      throw new InvalidArgumentError({
+        parameter: 'schema',
+        value: schema,
+        message: 'Schema is not supported for enum output.',
+      });
+    }
+
+    if (schemaDescription != null) {
+      throw new InvalidArgumentError({
+        parameter: 'schemaDescription',
+        value: schemaDescription,
+        message: 'Schema description is not supported for enum output.',
+      });
+    }
+
+    if (schemaName != null) {
+      throw new InvalidArgumentError({
+        parameter: 'schemaName',
+        value: schemaName,
+        message: 'Schema name is not supported for enum output.',
+      });
+    }
+
+    if (enumValues == null) {
+      throw new InvalidArgumentError({
+        parameter: 'enumValues',
+        value: enumValues,
+        message: 'Enum values are required for enum output.',
+      });
+    }
+
+    for (const value of enumValues) {
+      if (typeof value !== 'string') {
+        throw new InvalidArgumentError({
+          parameter: 'enumValues',
+          value,
+          message: 'Enum values must be strings.',
+        });
+      }
+    }
+  }
+}

package/src/generate-speech/generate-speech-result.ts

@@ -0,0 +1,30 @@
+import { JSONObject } from '@ai-sdk/provider';
+import { SpeechModelResponseMetadata } from '../types/speech-model-response-metadata';
+import { Warning } from '../types/warning';
+import { GeneratedAudioFile } from './generated-audio-file';
+
+/**
+ * The result of a `generateSpeech` call.
+ * It contains the audio data and additional information.
+ */
+export interface SpeechResult {
+  /**
+   * The generated audio file with the audio data.
+   */
+  readonly audio: GeneratedAudioFile;
+
+  /**
+   * Warnings for the call, e.g. unsupported settings.
+   */
+  readonly warnings: Array<Warning>;
+
+  /**
+   * Response metadata from the provider. There may be multiple responses if we made multiple calls to the model.
+   */
+  readonly responses: Array<SpeechModelResponseMetadata>;
+
+  /**
+   * Provider metadata from the provider.
+   */
+  readonly providerMetadata: Record<string, JSONObject>;
+}

package/src/generate-speech/generate-speech.ts

@@ -0,0 +1,191 @@
+import { JSONObject } from '@ai-sdk/provider';
+import { ProviderOptions, withUserAgentSuffix } from '@ai-sdk/provider-utils';
+import { NoSpeechGeneratedError } from '../error/no-speech-generated-error';
+import { logWarnings } from '../logger/log-warnings';
+import { resolveSpeechModel } from '../model/resolve-model';
+import { SpeechModel } from '../types/speech-model';
+import { SpeechModelResponseMetadata } from '../types/speech-model-response-metadata';
+import { Warning } from '../types/warning';
+import {
+  audioMediaTypeSignatures,
+  detectMediaType,
+} from '../util/detect-media-type';
+import { prepareRetries } from '../util/prepare-retries';
+import { VERSION } from '../version';
+import { SpeechResult } from './generate-speech-result';
+import {
+  DefaultGeneratedAudioFile,
+  GeneratedAudioFile,
+} from './generated-audio-file';
+
+/**
+ * Generates speech audio using a speech model.
+ *
+ * @param model - The speech model to use.
+ * @param text - The text to convert to speech.
+ * @param voice - The voice to use for speech generation.
+ * @param outputFormat - The output format to use for speech generation e.g. "mp3", "wav", etc.
+ * @param instructions - Instructions for the speech generation e.g. "Speak in a slow and steady tone".
+ * @param speed - The speed of the speech generation.
+ * @param language - The language for speech generation (ISO 639-1 code e.g. "en", "es", "fr") or "auto" for automatic detection.
+ * @param providerOptions - Additional provider-specific options that are passed through to the provider
+ * as body parameters.
+ * @param maxRetries - Maximum number of retries. Set to 0 to disable retries. Default: 2.
+ * @param abortSignal - An optional abort signal that can be used to cancel the call.
+ * @param headers - Additional HTTP headers to be sent with the request. Only applicable for HTTP-based providers.
+ *
+ * @returns A result object that contains the generated audio data.
+ */
+export async function generateSpeech({
+  model,
+  text,
+  voice,
+  outputFormat,
+  instructions,
+  speed,
+  language,
+  providerOptions = {},
+  maxRetries: maxRetriesArg,
+  abortSignal,
+  headers,
+}: {
+  /**
+   * The speech model to use.
+   */
+  model: SpeechModel;
+
+  /**
+   * The text to convert to speech.
+   */
+  text: string;
+
+  /**
+   * The voice to use for speech generation.
+   */
+  voice?: string;
+
+  /**
+   * The desired output format for the audio e.g. "mp3", "wav", etc.
+   */
+  outputFormat?: 'mp3' | 'wav' | (string & {});
+
+  /**
+   * Instructions for the speech generation e.g. "Speak in a slow and steady tone".
+   */
+  instructions?: string;
+
+  /**
+   * The speed of the speech generation.
+   */
+  speed?: number;
+
+  /**
+   * The language for speech generation. This should be an ISO 639-1 language code (e.g. "en", "es", "fr")
+   * or "auto" for automatic language detection. Provider support varies.
+   */
+  language?: string;
+
+  /**
+   * Additional provider-specific options that are passed through to the provider
+   * as body parameters.
+   *
+   * The outer record is keyed by the provider name, and the inner
+   * record is keyed by the provider-specific metadata key.
+   * ```ts
+   * {
+   * "openai": {}
+   * }
+   * ```
+   */
+  providerOptions?: ProviderOptions;
+
+  /**
+   * Maximum number of retries per speech model call. Set to 0 to disable retries.
+   *
+   * @default 2
+   */
+  maxRetries?: number;
+
+  /**
+   * Abort signal.
+   */
+  abortSignal?: AbortSignal;
+
+  /**
+   * Additional headers to include in the request.
+   * Only applicable for HTTP-based providers.
+   */
+  headers?: Record<string, string>;
+}): Promise<SpeechResult> {
+  const resolvedModel = resolveSpeechModel(model);
+  if (!resolvedModel) {
+    throw new Error('Model could not be resolved');
+  }
+
+  const headersWithUserAgent = withUserAgentSuffix(
+    headers ?? {},
+    `ai/${VERSION}`,
+  );
+
+  const { retry } = prepareRetries({
+    maxRetries: maxRetriesArg,
+    abortSignal,
+  });
+
+  const result = await retry(() =>
+    resolvedModel.doGenerate({
+      text,
+      voice,
+      outputFormat,
+      instructions,
+      speed,
+      language,
+      abortSignal,
+      headers: headersWithUserAgent,
+      providerOptions,
+    }),
+  );
+
+  if (!result.audio || result.audio.length === 0) {
+    throw new NoSpeechGeneratedError({ responses: [result.response] });
+  }
+
+  logWarnings({
+    warnings: result.warnings,
+    provider: resolvedModel.provider,
+    model: resolvedModel.modelId,
+  });
+
+  return new DefaultSpeechResult({
+    audio: new DefaultGeneratedAudioFile({
+      data: result.audio,
+      mediaType:
+        detectMediaType({
+          data: result.audio,
+          signatures: audioMediaTypeSignatures,
+        }) ?? 'audio/mp3',
+    }),
+    warnings: result.warnings,
+    responses: [result.response],
+    providerMetadata: result.providerMetadata,
+  });
+}
+
+class DefaultSpeechResult implements SpeechResult {
+  readonly audio: GeneratedAudioFile;
+  readonly warnings: Array<Warning>;
+  readonly responses: Array<SpeechModelResponseMetadata>;
+  readonly providerMetadata: Record<string, JSONObject>;
+
+  constructor(options: {
+    audio: GeneratedAudioFile;
+    warnings: Array<Warning>;
+    responses: Array<SpeechModelResponseMetadata>;
+    providerMetadata: Record<string, JSONObject> | undefined;
+  }) {
+    this.audio = options.audio;
+    this.warnings = options.warnings;
+    this.responses = options.responses;
+    this.providerMetadata = options.providerMetadata ?? {};
+  }
+}

package/src/generate-speech/generated-audio-file.ts

@@ -0,0 +1,65 @@
+import {
+  GeneratedFile,
+  DefaultGeneratedFile,
+} from '../generate-text/generated-file';
+
+/**
+ * A generated audio file.
+ */
+export interface GeneratedAudioFile extends GeneratedFile {
+  /**
+   * Audio format of the file (e.g., 'mp3', 'wav', etc.)
+   */
+  readonly format: string;
+}
+
+export class DefaultGeneratedAudioFile
+  extends DefaultGeneratedFile
+  implements GeneratedAudioFile
+{
+  readonly format: string;
+
+  constructor({
+    data,
+    mediaType,
+  }: {
+    data: string | Uint8Array;
+    mediaType: string;
+  }) {
+    super({ data, mediaType });
+    let format = 'mp3';
+
+    // If format is not provided, try to determine it from the media type
+    if (mediaType) {
+      const mediaTypeParts = mediaType.split('/');
+
+      if (mediaTypeParts.length === 2) {
+        // Handle special cases for audio formats
+        if (mediaType !== 'audio/mpeg') {
+          format = mediaTypeParts[1];
+        }
+      }
+    }
+
+    if (!format) {
+      // TODO this should be an AI SDK error
+      throw new Error(
+        'Audio format must be provided or determinable from media type',
+      );
+    }
+
+    this.format = format;
+  }
+}
+
+export class DefaultGeneratedAudioFileWithType extends DefaultGeneratedAudioFile {
+  readonly type = 'audio';
+
+  constructor(options: {
+    data: string | Uint8Array;
+    mediaType: string;
+    format: string;
+  }) {
+    super(options);
+  }
+}

package/src/generate-speech/index.ts

@@ -0,0 +1,3 @@
+export { generateSpeech as experimental_generateSpeech } from './generate-speech';
+export type { SpeechResult as Experimental_SpeechResult } from './generate-speech-result';
+export type { GeneratedAudioFile } from './generated-audio-file';

package/src/generate-text/collect-tool-approvals.ts

@@ -0,0 +1,116 @@
+import {
+  ModelMessage,
+  ToolApprovalRequest,
+  ToolApprovalResponse,
+} from '@ai-sdk/provider-utils';
+import { InvalidToolApprovalError } from '../error/invalid-tool-approval-error';
+import { ToolCallNotFoundForApprovalError } from '../error/tool-call-not-found-for-approval-error';
+import { TypedToolCall } from './tool-call';
+import { TypedToolResult } from './tool-result';
+import { ToolSet } from './tool-set';
+
+export type CollectedToolApprovals<TOOLS extends ToolSet> = {
+  approvalRequest: ToolApprovalRequest;
+  approvalResponse: ToolApprovalResponse;
+  toolCall: TypedToolCall<TOOLS>;
+};
+
+/**
+ * If the last message is a tool message, this function collects all tool approvals
+ * from that message.
+ */
+export function collectToolApprovals<TOOLS extends ToolSet>({
+  messages,
+}: {
+  messages: ModelMessage[];
+}): {
+  approvedToolApprovals: Array<CollectedToolApprovals<TOOLS>>;
+  deniedToolApprovals: Array<CollectedToolApprovals<TOOLS>>;
+} {
+  const lastMessage = messages.at(-1);
+
+  if (lastMessage?.role != 'tool') {
+    return {
+      approvedToolApprovals: [],
+      deniedToolApprovals: [],
+    };
+  }
+
+  // gather tool calls and prepare lookup
+  const toolCallsByToolCallId: Record<string, TypedToolCall<TOOLS>> = {};
+  for (const message of messages) {
+    if (message.role === 'assistant' && typeof message.content !== 'string') {
+      const content = message.content;
+      for (const part of content) {
+        if (part.type === 'tool-call') {
+          toolCallsByToolCallId[part.toolCallId] = part as TypedToolCall<TOOLS>;
+        }
+      }
+    }
+  }
+
+  // gather approval responses and prepare lookup
+  const toolApprovalRequestsByApprovalId: Record<string, ToolApprovalRequest> =
+    {};
+  for (const message of messages) {
+    if (message.role === 'assistant' && typeof message.content !== 'string') {
+      const content = message.content;
+      for (const part of content) {
+        if (part.type === 'tool-approval-request') {
+          toolApprovalRequestsByApprovalId[part.approvalId] = part;
+        }
+      }
+    }
+  }
+
+  // gather tool results from the last tool message
+  const toolResults: Record<string, TypedToolResult<TOOLS>> = {};
+  for (const part of lastMessage.content) {
+    if (part.type === 'tool-result') {
+      toolResults[part.toolCallId] = part as TypedToolResult<TOOLS>;
+    }
+  }
+
+  const approvedToolApprovals: Array<CollectedToolApprovals<TOOLS>> = [];
+  const deniedToolApprovals: Array<CollectedToolApprovals<TOOLS>> = [];
+
+  const approvalResponses = lastMessage.content.filter(
+    part => part.type === 'tool-approval-response',
+  );
+  for (const approvalResponse of approvalResponses) {
+    const approvalRequest =
+      toolApprovalRequestsByApprovalId[approvalResponse.approvalId];
+
+    if (approvalRequest == null) {
+      throw new InvalidToolApprovalError({
+        approvalId: approvalResponse.approvalId,
+      });
+    }
+
+    if (toolResults[approvalRequest.toolCallId] != null) {
+      continue;
+    }
+
+    const toolCall = toolCallsByToolCallId[approvalRequest.toolCallId];
+    if (toolCall == null) {
+      throw new ToolCallNotFoundForApprovalError({
+        toolCallId: approvalRequest.toolCallId,
+        approvalId: approvalRequest.approvalId,
+      });
+    }
+
+    const approval: CollectedToolApprovals<TOOLS> = {
+      approvalRequest,
+      approvalResponse,
+      toolCall,
+    };
+
+    if (approvalResponse.approved) {
+      approvedToolApprovals.push(approval);
+    } else {
+      deniedToolApprovals.push(approval);
+    }
+  }
+
+  return { approvedToolApprovals, deniedToolApprovals };
+}

package/src/generate-text/content-part.ts

@@ -0,0 +1,31 @@
+import { ProviderMetadata } from '../types';
+import { Source } from '../types/language-model';
+import { GeneratedFile } from './generated-file';
+import { ToolApprovalRequestOutput } from './tool-approval-request-output';
+import { ReasoningOutput, ReasoningFileOutput } from './reasoning-output';
+import { TypedToolCall } from './tool-call';
+import { TypedToolError } from './tool-error';
+import { TypedToolResult } from './tool-result';
+import { ToolSet } from './tool-set';
+
+export type ContentPart<TOOLS extends ToolSet> =
+  | { type: 'text'; text: string; providerMetadata?: ProviderMetadata }
+  | {
+      type: 'custom';
+      kind: `${string}.${string}`;
+      providerMetadata?: ProviderMetadata;
+    }
+  | ReasoningOutput
+  | ReasoningFileOutput
+  | ({ type: 'source' } & Source)
+  | { type: 'file'; file: GeneratedFile; providerMetadata?: ProviderMetadata }
+  | ({ type: 'tool-call' } & TypedToolCall<TOOLS> & {
+        providerMetadata?: ProviderMetadata;
+      })
+  | ({ type: 'tool-result' } & TypedToolResult<TOOLS> & {
+        providerMetadata?: ProviderMetadata;
+      })
+  | ({ type: 'tool-error' } & TypedToolError<TOOLS> & {
+        providerMetadata?: ProviderMetadata;
+      })
+  | ToolApprovalRequestOutput<TOOLS>;