@ai-sdk/google-vertex 4.0.189 → 4.0.193

package/src/gemini-transcription/google-vertex-gemini-transcription-model.ts

@@ -0,0 +1,253 @@
+import {
+  InvalidArgumentError,
+  type JSONObject,
+  type SharedV3Warning,
+  type TranscriptionModelV3,
+} from '@ai-sdk/provider';
+import {
+  combineHeaders,
+  convertToBase64,
+  createJsonResponseHandler,
+  parseProviderOptions,
+  postJsonToApi,
+  resolve,
+  type FetchFunction,
+  type Resolvable,
+} from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
+import { googleVertexFailedResponseHandler } from '../google-vertex-error';
+import {
+  googleVertexGeminiTranscriptionModelOptions,
+  type GoogleVertexGeminiTranscriptionModelId,
+  type GoogleVertexTranscriptionModelGeminiOptions,
+} from './google-vertex-gemini-transcription-model-options';
+
+/**
+ * Live transcription (`*-live` model variants) requires streaming support,
+ * which is only available in AI SDK v7 (transcription specification v4).
+ */
+function isLiveTranscriptionModelId(modelId: string): boolean {
+  return modelId.includes('-live');
+}
+
+interface GoogleVertexGeminiTranscriptionModelConfig {
+  provider: string;
+  /** Regional base URL ending in `/publishers/google`. */
+  baseURL: string;
+  headers?: Resolvable<Record<string, string | undefined>>;
+  fetch?: FetchFunction;
+  _internal?: {
+    currentDate?: () => Date;
+  };
+}
+
+/**
+ * Gemini transcription on Vertex AI (e.g. `gemini-3.5-transcribe`) via
+ * `generateContent` with `generationConfig.audioTranscriptionConfig`.
+ */
+export class GoogleVertexGeminiTranscriptionModel implements TranscriptionModelV3 {
+  readonly specificationVersion = 'v3';
+
+  get provider(): string {
+    return this.config.provider;
+  }
+
+  constructor(
+    readonly modelId: GoogleVertexGeminiTranscriptionModelId,
+    private readonly config: GoogleVertexGeminiTranscriptionModelConfig,
+  ) {}
+
+  private async parseOptions(
+    providerOptions: Record<string, unknown> | undefined,
+  ): Promise<GoogleVertexTranscriptionModelGeminiOptions | undefined> {
+    // The Vertex provider exposes options under `googleVertex`/`vertex`;
+    // accept `google` as a cross-namespace fallback (e.g. via the AI Gateway).
+    for (const provider of ['googleVertex', 'vertex', 'google'] as const) {
+      const parsed = await parseProviderOptions({
+        provider,
+        providerOptions,
+        schema: googleVertexGeminiTranscriptionModelOptions,
+      });
+      if (parsed != null) return parsed;
+    }
+    return undefined;
+  }
+
+  async doGenerate(
+    options: Parameters<TranscriptionModelV3['doGenerate']>[0],
+  ): Promise<Awaited<ReturnType<TranscriptionModelV3['doGenerate']>>> {
+    if (isLiveTranscriptionModelId(this.modelId)) {
+      throw new InvalidArgumentError({
+        argument: 'modelId',
+        message:
+          `Model '${this.modelId}' only supports streaming transcription, ` +
+          `which requires AI SDK v7. Use a unary model such as 'gemini-3.5-transcribe'.`,
+      });
+    }
+
+    const currentDate = this.config._internal?.currentDate?.() ?? new Date();
+    const warnings: SharedV3Warning[] = [];
+    const googleOptions = await this.parseOptions(options.providerOptions);
+    const audioTranscriptionConfig =
+      buildAudioTranscriptionConfig(googleOptions);
+
+    const requestBody = {
+      contents: [
+        {
+          role: 'user',
+          parts: [
+            {
+              inlineData: {
+                mimeType: options.mediaType,
+                data: convertToBase64(options.audio),
+              },
+            },
+          ],
+        },
+      ],
+      ...(audioTranscriptionConfig != null
+        ? { generationConfig: { audioTranscriptionConfig } }
+        : {}),
+    };
+
+    const {
+      value: response,
+      responseHeaders,
+      rawValue: rawResponse,
+    } = await postJsonToApi({
+      url: `${this.config.baseURL}/models/${this.modelId}:generateContent`,
+      headers: combineHeaders(
+        this.config.headers ? await resolve(this.config.headers) : undefined,
+        options.headers,
+      ),
+      body: requestBody,
+      failedResponseHandler: googleVertexFailedResponseHandler,
+      successfulResponseHandler: createJsonResponseHandler(
+        googleVertexGeminiTranscriptionResponseSchema,
+      ),
+      abortSignal: options.abortSignal,
+      fetch: this.config.fetch,
+    });
+
+    const parts = response.candidates?.[0]?.content?.parts ?? [];
+    const plainText = parts.map(part => part.text ?? '').join('');
+    const transcriptionText = parts
+      .map(part => part.audioTranscription?.text ?? '')
+      .join('');
+    const text = plainText !== '' ? plainText : transcriptionText;
+
+    let language: string | undefined;
+    const segments: Array<{
+      text: string;
+      startSecond: number;
+      endSecond: number;
+    }> = [];
+    for (const part of parts) {
+      const transcription = part.audioTranscription;
+      if (transcription == null) continue;
+      language ??= transcription.languageCode ?? undefined;
+      for (const word of transcription.words ?? []) {
+        const startSecond = parseOffsetSeconds(word.startOffset);
+        const endSecond = parseOffsetSeconds(word.endOffset);
+        if (word.word == null || startSecond == null || endSecond == null) {
+          continue;
+        }
+        segments.push({ text: word.word, startSecond, endSecond });
+      }
+    }
+
+    return {
+      text,
+      segments,
+      language,
+      durationInSeconds: undefined,
+      warnings,
+      response: {
+        timestamp: currentDate,
+        modelId: this.modelId,
+        headers: responseHeaders,
+        body: rawResponse,
+      },
+      ...(response.usageMetadata != null
+        ? {
+            providerMetadata: {
+              google: { usageMetadata: response.usageMetadata as JSONObject },
+            },
+          }
+        : {}),
+    };
+  }
+}
+
+/**
+ * Builds Google's `AudioTranscriptionConfig` from provider options; returns
+ * undefined when no options are set.
+ */
+function buildAudioTranscriptionConfig(
+  options: GoogleVertexTranscriptionModelGeminiOptions | undefined,
+): Record<string, unknown> | undefined {
+  if (options == null) return undefined;
+  const config: Record<string, unknown> = {};
+  if (options.languageCodes != null) {
+    config.languageCodes = options.languageCodes;
+  }
+  if (options.customVocabulary != null) {
+    config.customVocabulary = options.customVocabulary;
+  }
+  if (options.wordTimestamp != null) {
+    config.wordTimestamp = options.wordTimestamp;
+  }
+  if (options.diarization != null) {
+    config.diarization = options.diarization;
+  }
+  if (options.mode != null) {
+    config.mode = options.mode;
+  }
+  return Object.keys(config).length > 0 ? config : undefined;
+}
+
+/** Parses a Google duration offset such as `"1s"` or `"9.400s"` to seconds. */
+function parseOffsetSeconds(
+  offset: string | undefined | null,
+): number | undefined {
+  if (offset == null) return undefined;
+  const parsed = Number.parseFloat(offset);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}
+
+const googleVertexGeminiTranscriptionWordSchema = z.object({
+  word: z.string().nullish(),
+  startOffset: z.string().nullish(),
+  endOffset: z.string().nullish(),
+});
+
+const googleVertexGeminiTranscriptionResponseSchema = z.object({
+  candidates: z
+    .array(
+      z.object({
+        content: z
+          .object({
+            parts: z
+              .array(
+                z.object({
+                  text: z.string().nullish(),
+                  audioTranscription: z
+                    .object({
+                      text: z.string().nullish(),
+                      languageCode: z.string().nullish(),
+                      speakerLabel: z.string().nullish(),
+                      words: z
+                        .array(googleVertexGeminiTranscriptionWordSchema)
+                        .nullish(),
+                    })
+                    .nullish(),
+                }),
+              )
+              .nullish(),
+          })
+          .nullish(),
+      }),
+    )
+    .nullish(),
+  usageMetadata: z.record(z.string(), z.unknown()).nullish(),
+});

package/src/google-vertex-provider.ts

@@ -33,6 +33,7 @@ import type { GoogleVertexModelId } from './google-vertex-options';
 import { googleVertexTools } from './google-vertex-tools';
 import { GoogleVertexTranscriptionModel } from './google-vertex-transcription-model';
 import type { GoogleVertexTranscriptionModelId } from './google-vertex-transcription-model-options';
+import { GoogleVertexGeminiTranscriptionModel } from './gemini-transcription/google-vertex-gemini-transcription-model';
 import { GoogleVertexVideoModel } from './google-vertex-video-model';
 import type { GoogleVertexVideoModelId } from './google-vertex-video-settings';
 
@@ -305,6 +306,19 @@ export function createVertex(
     }
 
     const config = createConfig('transcription');
+
+    // Gemini transcription models (`gemini-3.5-transcribe`) use the Vertex
+    // generateContent surface; everything else routes to Cloud
+    // Speech-to-Text (Chirp, telephony).
+    if (modelId.startsWith('gemini')) {
+      return new GoogleVertexGeminiTranscriptionModel(modelId, {
+        provider: config.provider,
+        baseURL: loadBaseURL(),
+        headers: config.headers,
+        fetch: config.fetch,
+      });
+    }
+
     return new GoogleVertexTranscriptionModel(modelId, {
       provider: config.provider,
       headers: config.headers,

package/src/index.ts

@@ -14,6 +14,11 @@ export type {
   GoogleVertexVideoModelOptions as GoogleVertexVideoProviderOptions,
 } from './google-vertex-video-model';
 export type { GoogleVertexVideoModelId } from './google-vertex-video-settings';
+export { GoogleVertexGeminiTranscriptionModel } from './gemini-transcription/google-vertex-gemini-transcription-model';
+export type {
+  GoogleVertexGeminiTranscriptionModelId,
+  GoogleVertexTranscriptionModelGeminiOptions,
+} from './gemini-transcription/google-vertex-gemini-transcription-model-options';
 export { createVertex, vertex } from './google-vertex-provider-node';
 export type {
   GoogleVertexProvider,