ai 6.0.33 → 6.0.35

package/src/transcribe/transcribe.test.ts

@@ -0,0 +1,313 @@
+import { JSONObject, TranscriptionModelV3 } from '@ai-sdk/provider';
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  it,
+  vitest,
+  vi,
+} from 'vitest';
+import * as logWarningsModule from '../logger/log-warnings';
+import { MockTranscriptionModelV3 } from '../test/mock-transcription-model-v3';
+import { transcribe } from './transcribe';
+import { Warning } from '../types/warning';
+
+vi.mock('../version', () => {
+  return {
+    VERSION: '0.0.0-test',
+  };
+});
+
+const audioData = new Uint8Array([1, 2, 3, 4]); // Sample audio data
+const testDate = new Date(2024, 0, 1);
+
+const sampleTranscript = {
+  text: 'This is a sample transcript.',
+  segments: [
+    {
+      startSecond: 0,
+      endSecond: 2.5,
+      text: 'This is a',
+    },
+    {
+      startSecond: 2.5,
+      endSecond: 4.0,
+      text: 'sample transcript.',
+    },
+  ],
+  language: 'en',
+  durationInSeconds: 4.0,
+};
+
+const createMockResponse = (options: {
+  text: string;
+  segments: Array<{
+    text: string;
+    startSecond: number;
+    endSecond: number;
+  }>;
+  language?: string;
+  durationInSeconds?: number;
+  warnings?: Warning[];
+  timestamp?: Date;
+  modelId?: string;
+  headers?: Record<string, string>;
+  providerMetadata?: Record<string, JSONObject>;
+}) => ({
+  text: options.text,
+  segments: options.segments,
+  language: options.language,
+  durationInSeconds: options.durationInSeconds,
+  warnings: options.warnings ?? [],
+  response: {
+    timestamp: options.timestamp ?? new Date(),
+    modelId: options.modelId ?? 'test-model-id',
+    headers: options.headers ?? {},
+  },
+  providerMetadata: options.providerMetadata ?? {},
+});
+
+describe('transcribe', () => {
+  let logWarningsSpy: ReturnType<typeof vitest.spyOn>;
+
+  beforeEach(() => {
+    logWarningsSpy = vitest
+      .spyOn(logWarningsModule, 'logWarnings')
+      .mockImplementation(() => {});
+  });
+
+  afterEach(() => {
+    logWarningsSpy.mockRestore();
+  });
+
+  it('should send args to doGenerate', async () => {
+    const abortController = new AbortController();
+    const abortSignal = abortController.signal;
+
+    let capturedArgs!: Parameters<TranscriptionModelV3['doGenerate']>[0];
+
+    await transcribe({
+      model: new MockTranscriptionModelV3({
+        doGenerate: async args => {
+          capturedArgs = args;
+          return createMockResponse({
+            ...sampleTranscript,
+          });
+        },
+      }),
+      audio: audioData,
+      headers: {
+        'custom-request-header': 'request-header-value',
+      },
+      abortSignal,
+    });
+
+    expect(capturedArgs).toStrictEqual({
+      audio: audioData,
+      mediaType: 'audio/wav',
+      headers: {
+        'custom-request-header': 'request-header-value',
+        'user-agent': 'ai/0.0.0-test',
+      },
+      abortSignal,
+      providerOptions: {},
+    });
+  });
+
+  it('should return warnings', async () => {
+    const result = await transcribe({
+      model: new MockTranscriptionModelV3({
+        doGenerate: async () =>
+          createMockResponse({
+            ...sampleTranscript,
+            warnings: [
+              {
+                type: 'other',
+                message: 'Setting is not supported',
+              },
+            ],
+            providerMetadata: {
+              'test-provider': {
+                'test-key': 'test-value',
+              },
+            },
+          }),
+      }),
+      audio: audioData,
+    });
+
+    expect(result.warnings).toStrictEqual([
+      {
+        type: 'other',
+        message: 'Setting is not supported',
+      },
+    ]);
+  });
+
+  it('should call logWarnings with the correct warnings', async () => {
+    const expectedWarnings: Warning[] = [
+      {
+        type: 'other',
+        message: 'Setting is not supported',
+      },
+      {
+        type: 'unsupported',
+        feature: 'mediaType',
+        details: 'MediaType parameter not supported',
+      },
+    ];
+
+    await transcribe({
+      model: new MockTranscriptionModelV3({
+        doGenerate: async () =>
+          createMockResponse({
+            ...sampleTranscript,
+            warnings: expectedWarnings,
+          }),
+      }),
+      audio: audioData,
+    });
+
+    expect(logWarningsSpy).toHaveBeenCalledOnce();
+    expect(logWarningsSpy).toHaveBeenCalledWith({
+      warnings: expectedWarnings,
+      provider: 'mock-provider',
+      model: 'mock-model-id',
+    });
+  });
+
+  it('should call logWarnings with empty array when no warnings are present', async () => {
+    await transcribe({
+      model: new MockTranscriptionModelV3({
+        doGenerate: async () =>
+          createMockResponse({
+            ...sampleTranscript,
+            warnings: [], // no warnings
+          }),
+      }),
+      audio: audioData,
+    });
+
+    expect(logWarningsSpy).toHaveBeenCalledOnce();
+    expect(logWarningsSpy).toHaveBeenCalledWith({
+      warnings: [],
+      provider: 'mock-provider',
+      model: 'mock-model-id',
+    });
+  });
+
+  it('should return the transcript', async () => {
+    const result = await transcribe({
+      model: new MockTranscriptionModelV3({
+        doGenerate: async () =>
+          createMockResponse({
+            ...sampleTranscript,
+          }),
+      }),
+      audio: audioData,
+    });
+
+    expect(result).toEqual({
+      ...sampleTranscript,
+      warnings: [],
+      responses: [
+        {
+          timestamp: expect.any(Date),
+          modelId: 'test-model-id',
+          headers: {},
+        },
+      ],
+      providerMetadata: {},
+    });
+  });
+
+  describe('error handling', () => {
+    it('should throw NoTranscriptGeneratedError when no transcript is returned', async () => {
+      await expect(
+        transcribe({
+          model: new MockTranscriptionModelV3({
+            doGenerate: async () =>
+              createMockResponse({
+                text: '',
+                segments: [],
+                language: 'en',
+                durationInSeconds: 0,
+                timestamp: testDate,
+              }),
+          }),
+          audio: audioData,
+        }),
+      ).rejects.toMatchObject({
+        name: 'AI_NoTranscriptGeneratedError',
+        message: 'No transcript generated.',
+        responses: [
+          {
+            timestamp: testDate,
+            modelId: expect.any(String),
+          },
+        ],
+      });
+    });
+
+    it('should include response headers in error when no transcript generated', async () => {
+      await expect(
+        transcribe({
+          model: new MockTranscriptionModelV3({
+            doGenerate: async () =>
+              createMockResponse({
+                text: '',
+                segments: [],
+                language: 'en',
+                durationInSeconds: 0,
+                timestamp: testDate,
+                headers: {
+                  'custom-response-header': 'response-header-value',
+                  'user-agent': 'ai/0.0.0-test',
+                },
+              }),
+          }),
+          audio: audioData,
+        }),
+      ).rejects.toMatchObject({
+        name: 'AI_NoTranscriptGeneratedError',
+        message: 'No transcript generated.',
+        responses: [
+          {
+            timestamp: testDate,
+            modelId: expect.any(String),
+            headers: {
+              'custom-response-header': 'response-header-value',
+              'user-agent': 'ai/0.0.0-test',
+            },
+          },
+        ],
+      });
+    });
+  });
+
+  it('should return response metadata', async () => {
+    const testHeaders = { 'x-test': 'value' };
+
+    const result = await transcribe({
+      model: new MockTranscriptionModelV3({
+        doGenerate: async () =>
+          createMockResponse({
+            ...sampleTranscript,
+            timestamp: testDate,
+            modelId: 'test-model',
+            headers: testHeaders,
+          }),
+      }),
+      audio: audioData,
+    });
+
+    expect(result.responses).toStrictEqual([
+      {
+        timestamp: testDate,
+        modelId: 'test-model',
+        headers: testHeaders,
+      },
+    ]);
+  });
+});

package/src/transcribe/transcribe.ts

@@ -0,0 +1,173 @@
+import { JSONObject } from '@ai-sdk/provider';
+import { ProviderOptions, withUserAgentSuffix } from '@ai-sdk/provider-utils';
+import { NoTranscriptGeneratedError } from '../error/no-transcript-generated-error';
+import { logWarnings } from '../logger/log-warnings';
+import { DataContent } from '../prompt';
+import { convertDataContentToUint8Array } from '../prompt/data-content';
+import { TranscriptionModel } from '../types/transcription-model';
+import { TranscriptionModelResponseMetadata } from '../types/transcription-model-response-metadata';
+import {
+  audioMediaTypeSignatures,
+  detectMediaType,
+} from '../util/detect-media-type';
+import { download } from '../util/download/download';
+import { prepareRetries } from '../util/prepare-retries';
+import { TranscriptionResult } from './transcribe-result';
+import { VERSION } from '../version';
+import { resolveTranscriptionModel } from '../model/resolve-model';
+import { Warning } from '../types';
+/**
+Generates transcripts using a transcription model.
+
+@param model - The transcription model to use.
+@param audio - The audio data to transcribe as DataContent (string | Uint8Array | ArrayBuffer | Buffer) or a URL.
+@param providerOptions - Additional provider-specific options that are passed through to the provider
+as body parameters.
+@param maxRetries - Maximum number of retries. Set to 0 to disable retries. Default: 2.
+@param abortSignal - An optional abort signal that can be used to cancel the call.
+@param headers - Additional HTTP headers to be sent with the request. Only applicable for HTTP-based providers.
+
+@returns A result object that contains the generated transcript.
+ */
+export async function transcribe({
+  model,
+  audio,
+  providerOptions = {},
+  maxRetries: maxRetriesArg,
+  abortSignal,
+  headers,
+}: {
+  /**
+The transcription model to use.
+     */
+  model: TranscriptionModel;
+
+  /**
+The audio data to transcribe.
+   */
+  audio: DataContent | URL;
+
+  /**
+Additional provider-specific options that are passed through to the provider
+as body parameters.
+
+The outer record is keyed by the provider name, and the inner
+record is keyed by the provider-specific metadata key.
+```ts
+{
+  "openai": {
+    "temperature": 0
+  }
+}
+```
+     */
+  providerOptions?: ProviderOptions;
+
+  /**
+Maximum number of retries per transcript model call. Set to 0 to disable retries.
+
+@default 2
+   */
+  maxRetries?: number;
+
+  /**
+Abort signal.
+ */
+  abortSignal?: AbortSignal;
+
+  /**
+Additional headers to include in the request.
+Only applicable for HTTP-based providers.
+ */
+  headers?: Record<string, string>;
+}): Promise<TranscriptionResult> {
+  const resolvedModel = resolveTranscriptionModel(model);
+  if (!resolvedModel) {
+    throw new Error('Model could not be resolved');
+  }
+
+  const { retry } = prepareRetries({
+    maxRetries: maxRetriesArg,
+    abortSignal,
+  });
+
+  const headersWithUserAgent = withUserAgentSuffix(
+    headers ?? {},
+    `ai/${VERSION}`,
+  );
+
+  const audioData =
+    audio instanceof URL
+      ? (await download({ url: audio })).data
+      : convertDataContentToUint8Array(audio);
+
+  const result = await retry(() =>
+    resolvedModel.doGenerate({
+      audio: audioData,
+      abortSignal,
+      headers: headersWithUserAgent,
+      providerOptions,
+      mediaType:
+        detectMediaType({
+          data: audioData,
+          signatures: audioMediaTypeSignatures,
+        }) ?? 'audio/wav',
+    }),
+  );
+
+  logWarnings({
+    warnings: result.warnings,
+    provider: resolvedModel.provider,
+    model: resolvedModel.modelId,
+  });
+
+  if (!result.text) {
+    throw new NoTranscriptGeneratedError({ responses: [result.response] });
+  }
+
+  return new DefaultTranscriptionResult({
+    text: result.text,
+    segments: result.segments,
+    language: result.language,
+    durationInSeconds: result.durationInSeconds,
+    warnings: result.warnings,
+    responses: [result.response],
+    providerMetadata: result.providerMetadata,
+  });
+}
+
+class DefaultTranscriptionResult implements TranscriptionResult {
+  readonly text: string;
+  readonly segments: Array<{
+    text: string;
+    startSecond: number;
+    endSecond: number;
+  }>;
+  readonly language: string | undefined;
+  readonly durationInSeconds: number | undefined;
+  readonly warnings: Array<Warning>;
+  readonly responses: Array<TranscriptionModelResponseMetadata>;
+  readonly providerMetadata: Record<string, JSONObject>;
+
+  constructor(options: {
+    text: string;
+    segments: Array<{
+      text: string;
+      startSecond: number;
+      endSecond: number;
+    }>;
+    language: string | undefined;
+    durationInSeconds: number | undefined;
+    warnings: Array<Warning>;
+    responses: Array<TranscriptionModelResponseMetadata>;
+    providerMetadata: Record<string, JSONObject> | undefined;
+  }) {
+    this.text = options.text;
+    this.segments = options.segments;
+    this.language = options.language;
+    this.durationInSeconds = options.durationInSeconds;
+    this.warnings = options.warnings;
+    this.responses = options.responses;
+    this.providerMetadata = options.providerMetadata ?? {};
+  }
+}

package/src/types/embedding-model-middleware.ts

@@ -0,0 +1,3 @@
+import { EmbeddingModelV3Middleware } from '@ai-sdk/provider';
+
+export type EmbeddingModelMiddleware = EmbeddingModelV3Middleware;

package/src/types/embedding-model.ts

@@ -0,0 +1,18 @@
+import {
+  EmbeddingModelV2,
+  EmbeddingModelV3,
+  EmbeddingModelV3Embedding,
+} from '@ai-sdk/provider';
+
+/**
+Embedding model that is used by the AI SDK.
+*/
+export type EmbeddingModel =
+  | string
+  | EmbeddingModelV3
+  | EmbeddingModelV2<string>;
+
+/**
+Embedding.
+ */
+export type Embedding = EmbeddingModelV3Embedding;

package/src/types/image-model-middleware.ts

@@ -0,0 +1,3 @@
+import { ImageModelV3Middleware } from '@ai-sdk/provider';
+
+export type ImageModelMiddleware = ImageModelV3Middleware;

package/src/types/image-model-response-metadata.ts

@@ -0,0 +1,16 @@
+export type ImageModelResponseMetadata = {
+  /**
+Timestamp for the start of the generated response.
+   */
+  timestamp: Date;
+
+  /**
+The ID of the response model that was used to generate the response.
+   */
+  modelId: string;
+
+  /**
+Response headers.
+   */
+  headers?: Record<string, string>;
+};

package/src/types/image-model.ts

@@ -0,0 +1,19 @@
+import {
+  ImageModelV2,
+  ImageModelV3,
+  ImageModelV3ProviderMetadata,
+  ImageModelV2ProviderMetadata,
+} from '@ai-sdk/provider';
+
+/**
+Image model that is used by the AI SDK.
+  */
+export type ImageModel = string | ImageModelV3 | ImageModelV2;
+
+/**
+Metadata from the model provider for this call
+  */
+// TODO should this be v3 only?
+export type ImageModelProviderMetadata =
+  | ImageModelV3ProviderMetadata
+  | ImageModelV2ProviderMetadata;

package/src/types/index.ts

@@ -0,0 +1,29 @@
+export type { JSONSchema7 } from '@ai-sdk/provider';
+export type { Embedding, EmbeddingModel } from './embedding-model';
+export type { EmbeddingModelMiddleware } from './embedding-model-middleware';
+export type { ImageModel, ImageModelProviderMetadata } from './image-model';
+export type { ImageModelMiddleware } from './image-model-middleware';
+export type { ImageModelResponseMetadata } from './image-model-response-metadata';
+export type { JSONValue } from './json-value';
+export type {
+  CallWarning,
+  FinishReason,
+  LanguageModel,
+  ToolChoice,
+} from './language-model';
+export type { LanguageModelMiddleware } from './language-model-middleware';
+export type { LanguageModelRequestMetadata } from './language-model-request-metadata';
+export type { LanguageModelResponseMetadata } from './language-model-response-metadata';
+export type { Provider } from './provider';
+export type { ProviderMetadata } from './provider-metadata';
+export type { RerankingModel } from './reranking-model';
+export type { SpeechModel } from './speech-model';
+export type { SpeechModelResponseMetadata } from './speech-model-response-metadata';
+export type { TranscriptionModel } from './transcription-model';
+export type { TranscriptionModelResponseMetadata } from './transcription-model-response-metadata';
+export type {
+  EmbeddingModelUsage,
+  ImageModelUsage,
+  LanguageModelUsage,
+} from './usage';
+export type { Warning } from './warning';

package/src/types/json-value.ts

@@ -0,0 +1,15 @@
+import { JSONValue as OriginalJSONValue } from '@ai-sdk/provider';
+import { z } from 'zod/v4';
+
+export const jsonValueSchema: z.ZodType<JSONValue> = z.lazy(() =>
+  z.union([
+    z.null(),
+    z.string(),
+    z.number(),
+    z.boolean(),
+    z.record(z.string(), jsonValueSchema.optional()),
+    z.array(jsonValueSchema),
+  ]),
+);
+
+export type JSONValue = OriginalJSONValue;

package/src/types/language-model-middleware.ts

@@ -0,0 +1,3 @@
+import { LanguageModelV3Middleware } from '@ai-sdk/provider';
+
+export type LanguageModelMiddleware = LanguageModelV3Middleware;

package/src/types/language-model-request-metadata.ts

@@ -0,0 +1,6 @@
+export type LanguageModelRequestMetadata = {
+  /**
+Request HTTP body that was sent to the provider API.
+     */
+  body?: unknown;
+};

package/src/types/language-model-response-metadata.ts

@@ -0,0 +1,21 @@
+export type LanguageModelResponseMetadata = {
+  /**
+  ID for the generated response.
+     */
+  id: string;
+
+  /**
+  Timestamp for the start of the generated response.
+  */
+  timestamp: Date;
+
+  /**
+  The ID of the response model that was used to generate the response.
+  */
+  modelId: string;
+
+  /**
+Response headers (available only for providers that use HTTP requests).
+     */
+  headers?: Record<string, string>;
+};