@fondation-io/ai 7.0.0-beta.45

package/src/test/mock-tracer.ts

@@ -0,0 +1,156 @@
+import {
+  AttributeValue,
+  Attributes,
+  Context,
+  Exception,
+  Span,
+  SpanContext,
+  SpanOptions,
+  SpanStatus,
+  TimeInput,
+  Tracer,
+} from '@opentelemetry/api';
+
+export class MockTracer implements Tracer {
+  spans: MockSpan[] = [];
+
+  get jsonSpans() {
+    return this.spans.map(span => ({
+      name: span.name,
+      attributes: span.attributes,
+      events: span.events,
+      ...(span.status && { status: span.status }),
+    }));
+  }
+
+  startSpan(name: string, options?: SpanOptions, context?: Context): Span {
+    const span = new MockSpan({
+      name,
+      options,
+      context,
+    });
+    this.spans.push(span);
+    return span;
+  }
+
+  startActiveSpan<F extends (span: Span) => unknown>(
+    name: string,
+    arg1: unknown,
+    arg2?: unknown,
+    arg3?: F,
+  ): ReturnType<any> {
+    if (typeof arg1 === 'function') {
+      const span = new MockSpan({
+        name,
+      });
+      this.spans.push(span);
+      return arg1(span);
+    }
+    if (typeof arg2 === 'function') {
+      const span = new MockSpan({
+        name,
+        options: arg1 as SpanOptions,
+      });
+      this.spans.push(span);
+      return arg2(span);
+    }
+    if (typeof arg3 === 'function') {
+      const span = new MockSpan({
+        name,
+        options: arg1 as SpanOptions,
+        context: arg2 as Context,
+      });
+      this.spans.push(span);
+      return arg3(span);
+    }
+  }
+}
+
+class MockSpan implements Span {
+  name: string;
+  context?: Context;
+  options?: SpanOptions;
+  attributes: Attributes;
+  events: Array<{
+    name: string;
+    attributes: Attributes | undefined;
+    time?: [number, number];
+  }> = [];
+  status?: SpanStatus;
+
+  readonly _spanContext: SpanContext = new MockSpanContext();
+
+  constructor({
+    name,
+    options,
+    context,
+  }: {
+    name: string;
+    options?: SpanOptions;
+    context?: Context;
+  }) {
+    this.name = name;
+    this.context = context;
+    this.options = options;
+    this.attributes = options?.attributes ?? {};
+  }
+
+  spanContext(): SpanContext {
+    return this._spanContext;
+  }
+
+  setAttribute(key: string, value: AttributeValue): this {
+    this.attributes = { ...this.attributes, [key]: value };
+    return this;
+  }
+
+  setAttributes(attributes: Attributes): this {
+    this.attributes = { ...this.attributes, ...attributes };
+    return this;
+  }
+
+  addEvent(name: string, attributes?: Attributes): this {
+    this.events.push({ name, attributes });
+    return this;
+  }
+
+  addLink() {
+    return this;
+  }
+  addLinks() {
+    return this;
+  }
+  setStatus(status: SpanStatus) {
+    this.status = status;
+    return this;
+  }
+  updateName() {
+    return this;
+  }
+  end() {
+    return this;
+  }
+  isRecording() {
+    return false;
+  }
+  recordException(exception: Exception, time?: TimeInput) {
+    const error =
+      typeof exception === 'string' ? new Error(exception) : exception;
+    this.events.push({
+      name: 'exception',
+      attributes: {
+        'exception.type': error.constructor?.name || 'Error',
+        'exception.name': error.name || 'Error',
+        'exception.message': error.message || '',
+        'exception.stack': error.stack || '',
+      },
+      time: Array.isArray(time) ? time : [0, 0],
+    });
+  }
+}
+
+class MockSpanContext implements SpanContext {
+  traceId = 'test-trace-id';
+  spanId = 'test-span-id';
+  traceFlags = 0;
+}

package/src/test/mock-transcription-model-v2.ts

@@ -0,0 +1,24 @@
+import { TranscriptionModelV2 } from '@ai-sdk/provider';
+import { notImplemented } from './not-implemented';
+
+export class MockTranscriptionModelV2 implements TranscriptionModelV2 {
+  readonly specificationVersion = 'v2';
+  readonly provider: TranscriptionModelV2['provider'];
+  readonly modelId: TranscriptionModelV2['modelId'];
+
+  doGenerate: TranscriptionModelV2['doGenerate'];
+
+  constructor({
+    provider = 'mock-provider',
+    modelId = 'mock-model-id',
+    doGenerate = notImplemented,
+  }: {
+    provider?: TranscriptionModelV2['provider'];
+    modelId?: TranscriptionModelV2['modelId'];
+    doGenerate?: TranscriptionModelV2['doGenerate'];
+  } = {}) {
+    this.provider = provider;
+    this.modelId = modelId;
+    this.doGenerate = doGenerate;
+  }
+}

package/src/test/mock-transcription-model-v3.ts

@@ -0,0 +1,24 @@
+import { TranscriptionModelV3 } from '@ai-sdk/provider';
+import { notImplemented } from './not-implemented';
+
+export class MockTranscriptionModelV3 implements TranscriptionModelV3 {
+  readonly specificationVersion = 'v3';
+  readonly provider: TranscriptionModelV3['provider'];
+  readonly modelId: TranscriptionModelV3['modelId'];
+
+  doGenerate: TranscriptionModelV3['doGenerate'];
+
+  constructor({
+    provider = 'mock-provider',
+    modelId = 'mock-model-id',
+    doGenerate = notImplemented,
+  }: {
+    provider?: TranscriptionModelV3['provider'];
+    modelId?: TranscriptionModelV3['modelId'];
+    doGenerate?: TranscriptionModelV3['doGenerate'];
+  } = {}) {
+    this.provider = provider;
+    this.modelId = modelId;
+    this.doGenerate = doGenerate;
+  }
+}

package/src/test/mock-transcription-model-v4.ts

@@ -0,0 +1,24 @@
+import { TranscriptionModelV4 } from '@ai-sdk/provider';
+import { notImplemented } from './not-implemented';
+
+export class MockTranscriptionModelV4 implements TranscriptionModelV4 {
+  readonly specificationVersion = 'v4';
+  readonly provider: TranscriptionModelV4['provider'];
+  readonly modelId: TranscriptionModelV4['modelId'];
+
+  doGenerate: TranscriptionModelV4['doGenerate'];
+
+  constructor({
+    provider = 'mock-provider',
+    modelId = 'mock-model-id',
+    doGenerate = notImplemented,
+  }: {
+    provider?: TranscriptionModelV4['provider'];
+    modelId?: TranscriptionModelV4['modelId'];
+    doGenerate?: TranscriptionModelV4['doGenerate'];
+  } = {}) {
+    this.provider = provider;
+    this.modelId = modelId;
+    this.doGenerate = doGenerate;
+  }
+}

package/src/test/mock-values.ts

@@ -0,0 +1,4 @@
+export function mockValues<T>(...values: T[]): () => T {
+  let counter = 0;
+  return () => values[counter++] ?? values[values.length - 1];
+}

package/src/test/mock-video-model-v3.ts

@@ -0,0 +1,28 @@
+import { Experimental_VideoModelV3 } from '@ai-sdk/provider';
+import { notImplemented } from './not-implemented';
+
+export class MockVideoModelV3 implements Experimental_VideoModelV3 {
+  readonly specificationVersion = 'v3';
+  readonly provider: Experimental_VideoModelV3['provider'];
+  readonly modelId: Experimental_VideoModelV3['modelId'];
+  readonly maxVideosPerCall: Experimental_VideoModelV3['maxVideosPerCall'];
+
+  doGenerate: Experimental_VideoModelV3['doGenerate'];
+
+  constructor({
+    provider = 'mock-provider',
+    modelId = 'mock-model-id',
+    maxVideosPerCall = 1,
+    doGenerate = notImplemented,
+  }: {
+    provider?: Experimental_VideoModelV3['provider'];
+    modelId?: Experimental_VideoModelV3['modelId'];
+    maxVideosPerCall?: Experimental_VideoModelV3['maxVideosPerCall'];
+    doGenerate?: Experimental_VideoModelV3['doGenerate'];
+  } = {}) {
+    this.provider = provider;
+    this.modelId = modelId;
+    this.maxVideosPerCall = maxVideosPerCall;
+    this.doGenerate = doGenerate;
+  }
+}

package/src/test/mock-video-model-v4.ts

@@ -0,0 +1,28 @@
+import { Experimental_VideoModelV4 } from '@ai-sdk/provider';
+import { notImplemented } from './not-implemented';
+
+export class MockVideoModelV4 implements Experimental_VideoModelV4 {
+  readonly specificationVersion = 'v4';
+  readonly provider: Experimental_VideoModelV4['provider'];
+  readonly modelId: Experimental_VideoModelV4['modelId'];
+  readonly maxVideosPerCall: Experimental_VideoModelV4['maxVideosPerCall'];
+
+  doGenerate: Experimental_VideoModelV4['doGenerate'];
+
+  constructor({
+    provider = 'mock-provider',
+    modelId = 'mock-model-id',
+    maxVideosPerCall = 1,
+    doGenerate = notImplemented,
+  }: {
+    provider?: Experimental_VideoModelV4['provider'];
+    modelId?: Experimental_VideoModelV4['modelId'];
+    maxVideosPerCall?: Experimental_VideoModelV4['maxVideosPerCall'];
+    doGenerate?: Experimental_VideoModelV4['doGenerate'];
+  } = {}) {
+    this.provider = provider;
+    this.modelId = modelId;
+    this.maxVideosPerCall = maxVideosPerCall;
+    this.doGenerate = doGenerate;
+  }
+}

package/src/test/not-implemented.ts

@@ -0,0 +1,3 @@
+export function notImplemented(): never {
+  throw new Error('Not implemented');
+}

package/src/text-stream/create-text-stream-response.ts

@@ -0,0 +1,30 @@
+import { prepareHeaders } from '../util/prepare-headers';
+
+/**
+ * Creates a Response object from a text stream.
+ * Each text chunk is encoded as UTF-8 and sent as a separate chunk.
+ * Sets a `Content-Type` header to `text/plain; charset=utf-8`.
+ *
+ * @param options - The options for creating the response.
+ * @param options.status - Optional HTTP status code (default: 200).
+ * @param options.statusText - Optional HTTP status text.
+ * @param options.headers - Optional response headers.
+ * @param options.textStream - The text stream to send.
+ * @returns A Response object with the text stream body.
+ */
+export function createTextStreamResponse({
+  status,
+  statusText,
+  headers,
+  textStream,
+}: ResponseInit & {
+  textStream: ReadableStream<string>;
+}): Response {
+  return new Response(textStream.pipeThrough(new TextEncoderStream()), {
+    status: status ?? 200,
+    statusText,
+    headers: prepareHeaders(headers, {
+      'content-type': 'text/plain; charset=utf-8',
+    }),
+  });
+}

package/src/text-stream/index.ts

@@ -0,0 +1,2 @@
+export { createTextStreamResponse } from './create-text-stream-response';
+export { pipeTextStreamToResponse } from './pipe-text-stream-to-response';

package/src/text-stream/pipe-text-stream-to-response.ts

@@ -0,0 +1,38 @@
+import { ServerResponse } from 'node:http';
+import { prepareHeaders } from '../util/prepare-headers';
+import { writeToServerResponse } from '../util/write-to-server-response';
+
+/**
+ * Writes a text stream to a Node.js ServerResponse object.
+ * Each text chunk is encoded as UTF-8 and written as a separate chunk.
+ * Sets a `Content-Type` header to `text/plain; charset=utf-8`.
+ *
+ * @param options - The options for piping the stream.
+ * @param options.response - The Node.js ServerResponse to write to.
+ * @param options.status - Optional HTTP status code.
+ * @param options.statusText - Optional HTTP status text.
+ * @param options.headers - Optional response headers.
+ * @param options.textStream - The text stream to pipe.
+ */
+export function pipeTextStreamToResponse({
+  response,
+  status,
+  statusText,
+  headers,
+  textStream,
+}: {
+  response: ServerResponse;
+  textStream: ReadableStream<string>;
+} & ResponseInit): void {
+  writeToServerResponse({
+    response,
+    status,
+    statusText,
+    headers: Object.fromEntries(
+      prepareHeaders(headers, {
+        'content-type': 'text/plain; charset=utf-8',
+      }).entries(),
+    ),
+    stream: textStream.pipeThrough(new TextEncoderStream()),
+  });
+}

package/src/transcribe/index.ts

@@ -0,0 +1,2 @@
+export { transcribe as experimental_transcribe } from './transcribe';
+export type { TranscriptionResult as Experimental_TranscriptionResult } from './transcribe-result';

package/src/transcribe/transcribe-result.ts

@@ -0,0 +1,60 @@
+import { JSONObject } from '@ai-sdk/provider';
+import { TranscriptionModelResponseMetadata } from '../types/transcription-model-response-metadata';
+import { Warning } from '../types/warning';
+
+/**
+ * The result of a `transcribe` call.
+ * It contains the transcript and additional information.
+ */
+export interface TranscriptionResult {
+  /**
+   * The complete transcribed text from the audio.
+   */
+  readonly text: string;
+
+  /**
+   * Array of transcript segments with timing information.
+   * Each segment represents a portion of the transcribed text with start and end times.
+   */
+  readonly segments: Array<{
+    /**
+     * The text content of this segment.
+     */
+    readonly text: string;
+    /**
+     * The start time of this segment in seconds.
+     */
+    readonly startSecond: number;
+    /**
+     * The end time of this segment in seconds.
+     */
+    readonly endSecond: number;
+  }>;
+
+  /**
+   * The detected language of the audio content, as an ISO-639-1 code (e.g., 'en' for English).
+   * May be undefined if the language couldn't be detected.
+   */
+  readonly language: string | undefined;
+
+  /**
+   * The total duration of the audio file in seconds.
+   * May be undefined if the duration couldn't be determined.
+   */
+  readonly durationInSeconds: number | undefined;
+
+  /**
+   * Warnings for the call, e.g. unsupported settings.
+   */
+  readonly warnings: Array<Warning>;
+
+  /**
+   * Response metadata from the provider. There may be multiple responses if we made multiple calls to the model.
+   */
+  readonly responses: Array<TranscriptionModelResponseMetadata>;
+
+  /**
+   * Provider metadata from the provider.
+   */
+  readonly providerMetadata: Record<string, JSONObject>;
+}

package/src/transcribe/transcribe.ts

@@ -0,0 +1,187 @@
+import { JSONObject } from '@ai-sdk/provider';
+import { ProviderOptions, withUserAgentSuffix } from '@ai-sdk/provider-utils';
+import { NoTranscriptGeneratedError } from '../error/no-transcript-generated-error';
+import { logWarnings } from '../logger/log-warnings';
+import { DataContent } from '../prompt';
+import { convertDataContentToUint8Array } from '../prompt/data-content';
+import { TranscriptionModel } from '../types/transcription-model';
+import { TranscriptionModelResponseMetadata } from '../types/transcription-model-response-metadata';
+import {
+  audioMediaTypeSignatures,
+  detectMediaType,
+} from '../util/detect-media-type';
+import { createDownload } from '../util/download/create-download';
+import { prepareRetries } from '../util/prepare-retries';
+import { TranscriptionResult } from './transcribe-result';
+import { VERSION } from '../version';
+import { resolveTranscriptionModel } from '../model/resolve-model';
+import { Warning } from '../types';
+/**
+ * Generates transcripts using a transcription model.
+ *
+ * @param model - The transcription model to use.
+ * @param audio - The audio data to transcribe as DataContent (string | Uint8Array | ArrayBuffer | Buffer) or a URL.
+ * @param providerOptions - Additional provider-specific options that are passed through to the provider
+ * as body parameters.
+ * @param maxRetries - Maximum number of retries. Set to 0 to disable retries. Default: 2.
+ * @param abortSignal - An optional abort signal that can be used to cancel the call.
+ * @param headers - Additional HTTP headers to be sent with the request. Only applicable for HTTP-based providers.
+ *
+ * @returns A result object that contains the generated transcript.
+ */
+const defaultDownload = createDownload();
+
+export async function transcribe({
+  model,
+  audio,
+  providerOptions = {},
+  maxRetries: maxRetriesArg,
+  abortSignal,
+  headers,
+  download: downloadFn = defaultDownload,
+}: {
+  /**
+   * The transcription model to use.
+   */
+  model: TranscriptionModel;
+
+  /**
+   * The audio data to transcribe.
+   */
+  audio: DataContent | URL;
+
+  /**
+   * Additional provider-specific options that are passed through to the provider
+   * as body parameters.
+   *
+   * The outer record is keyed by the provider name, and the inner
+   * record is keyed by the provider-specific metadata key.
+   * ```ts
+   * {
+   *   "openai": {
+   *     "temperature": 0
+   *   }
+   * }
+   * ```
+   */
+  providerOptions?: ProviderOptions;
+
+  /**
+   * Maximum number of retries per transcript model call. Set to 0 to disable retries.
+   *
+   * @default 2
+   */
+  maxRetries?: number;
+
+  /**
+   * Abort signal.
+   */
+  abortSignal?: AbortSignal;
+
+  /**
+   * Additional headers to include in the request.
+   * Only applicable for HTTP-based providers.
+   */
+  headers?: Record<string, string>;
+
+  /**
+   * Custom download function for fetching audio from URLs.
+   * Use `createDownload()` from `ai` to create a download function with custom size limits.
+   *
+   * @default createDownload() (2 GiB limit)
+   */
+  download?: (options: {
+    url: URL;
+    abortSignal?: AbortSignal;
+  }) => Promise<{ data: Uint8Array; mediaType: string | undefined }>;
+}): Promise<TranscriptionResult> {
+  const resolvedModel = resolveTranscriptionModel(model);
+  if (!resolvedModel) {
+    throw new Error('Model could not be resolved');
+  }
+
+  const { retry } = prepareRetries({
+    maxRetries: maxRetriesArg,
+    abortSignal,
+  });
+
+  const headersWithUserAgent = withUserAgentSuffix(
+    headers ?? {},
+    `ai/${VERSION}`,
+  );
+
+  const audioData =
+    audio instanceof URL
+      ? (await downloadFn({ url: audio, abortSignal })).data
+      : convertDataContentToUint8Array(audio);
+
+  const result = await retry(() =>
+    resolvedModel.doGenerate({
+      audio: audioData,
+      abortSignal,
+      headers: headersWithUserAgent,
+      providerOptions,
+      mediaType:
+        detectMediaType({
+          data: audioData,
+          signatures: audioMediaTypeSignatures,
+        }) ?? 'audio/wav',
+    }),
+  );
+
+  logWarnings({
+    warnings: result.warnings,
+    provider: resolvedModel.provider,
+    model: resolvedModel.modelId,
+  });
+
+  if (!result.text) {
+    throw new NoTranscriptGeneratedError({ responses: [result.response] });
+  }
+
+  return new DefaultTranscriptionResult({
+    text: result.text,
+    segments: result.segments,
+    language: result.language,
+    durationInSeconds: result.durationInSeconds,
+    warnings: result.warnings,
+    responses: [result.response],
+    providerMetadata: result.providerMetadata,
+  });
+}
+
+class DefaultTranscriptionResult implements TranscriptionResult {
+  readonly text: string;
+  readonly segments: Array<{
+    text: string;
+    startSecond: number;
+    endSecond: number;
+  }>;
+  readonly language: string | undefined;
+  readonly durationInSeconds: number | undefined;
+  readonly warnings: Array<Warning>;
+  readonly responses: Array<TranscriptionModelResponseMetadata>;
+  readonly providerMetadata: Record<string, JSONObject>;
+
+  constructor(options: {
+    text: string;
+    segments: Array<{
+      text: string;
+      startSecond: number;
+      endSecond: number;
+    }>;
+    language: string | undefined;
+    durationInSeconds: number | undefined;
+    warnings: Array<Warning>;
+    responses: Array<TranscriptionModelResponseMetadata>;
+    providerMetadata: Record<string, JSONObject> | undefined;
+  }) {
+    this.text = options.text;
+    this.segments = options.segments;
+    this.language = options.language;
+    this.durationInSeconds = options.durationInSeconds;
+    this.warnings = options.warnings;
+    this.responses = options.responses;
+    this.providerMetadata = options.providerMetadata ?? {};
+  }
+}

package/src/types/embedding-model-middleware.ts

@@ -0,0 +1,15 @@
+import { EmbeddingModelV4Middleware } from '@ai-sdk/provider';
+
+/**
+ * Middleware for embedding models.
+ * Accepts both V3 and V4 middleware types for backward compatibility.
+ *
+ * Uses EmbeddingModelV4Middleware as the base but relaxes specificationVersion
+ * to accept any string (including 'v3') and makes it optional.
+ */
+export type EmbeddingModelMiddleware = Omit<
+  EmbeddingModelV4Middleware,
+  'specificationVersion'
+> & {
+  readonly specificationVersion?: string;
+};

package/src/types/embedding-model.ts

@@ -0,0 +1,20 @@
+import {
+  EmbeddingModelV2,
+  EmbeddingModelV3,
+  EmbeddingModelV4,
+  EmbeddingModelV4Embedding,
+} from '@ai-sdk/provider';
+
+/**
+ * Embedding model that is used by the AI SDK.
+ */
+export type EmbeddingModel =
+  | string
+  | EmbeddingModelV4
+  | EmbeddingModelV3
+  | EmbeddingModelV2<string>;
+
+/**
+ * Embedding.
+ */
+export type Embedding = EmbeddingModelV4Embedding;

package/src/types/image-model-middleware.ts

@@ -0,0 +1,15 @@
+import { ImageModelV4Middleware } from '@ai-sdk/provider';
+
+/**
+ * Middleware for image models.
+ * Accepts both V3 and V4 middleware types for backward compatibility.
+ *
+ * Uses ImageModelV4Middleware as the base but relaxes specificationVersion
+ * to accept any string (including 'v3') and makes it optional.
+ */
+export type ImageModelMiddleware = Omit<
+  ImageModelV4Middleware,
+  'specificationVersion'
+> & {
+  readonly specificationVersion?: string;
+};