@zhivex-ai/gemini 0.10.5 → 0.11.1

package/dist/index.js

@@ -1,4 +1,5 @@
 import { toJSONSchema } from "zod";
+import { capabilities, groundedCapabilities, imageGenerationCapabilities, isGeminiLiveTranslateModel, isGeminiLiveTranscribeModel, musicGenerationCapabilities, realtimeCapabilities, speechCapabilities, transcriptionCapabilities, videoGenerationCapabilities, } from "./capabilities.js";
 import { CallbackRealtimeSession, ConfigurationError, ProviderHTTPError, UnsupportedFeatureError, assertTrustedEndpoint, createMcpToolSet, createProviderAdapter, decodeBase64WithLimit, encodeAudioFrame, encodeMediaFrame, isCallableToolDefinition, isHostedToolDefinition, normalizeFinishReason, openWebSocketConnection, readBodyWithLimit, readErrorBodyWithLimit, readJsonWithLimit, streamSSE, toToolSet, toolResultPayload, withRetry, withTimeoutSignal } from "@zhivex-ai/core";
 const encodeGeminiPathSegment = (value, label) => {
     if (!value || value === "." || value === ".." || /[\\/?#\s]/.test(value)) {
@@ -16,196 +17,6 @@ const encodeGeminiResourceName = (value, label) => {
     }
     return segments.map(encodeURIComponent).join("/");
 };
-const capabilities = {
-    streaming: true,
-    tools: true,
-    structuredOutput: true,
-    jsonMode: true,
-    toolChoice: true,
-    parallelToolCalls: false,
-    vision: true,
-    files: true,
-    audioInput: true,
-    audioOutput: false,
-    embeddings: true,
-    fileSearch: true,
-    urlContext: true,
-    contextCaching: true,
-    batch: true,
-    interactions: true,
-    rawPrediction: true,
-    computerUse: true,
-    reasoning: true,
-    webSearch: true,
-    agentCapabilities: {
-        supportTier: "tier-b",
-        toolChoiceNone: true,
-        approvalRequests: false,
-        hostedWebSearch: true,
-        hostedFileSearch: true,
-        remoteMcp: false,
-        computerUse: true,
-        codeExecution: true,
-        toolsets: false
-    }
-};
-const transcriptionCapabilities = {
-    ...capabilities,
-    streaming: false,
-    tools: false,
-    structuredOutput: false,
-    jsonMode: false,
-    toolChoice: false,
-    parallelToolCalls: false,
-    audioInput: true,
-    audioOutput: false,
-    embeddings: false,
-    reasoning: false,
-    webSearch: false,
-    agentCapabilities: {
-        supportTier: "tier-c",
-        toolChoiceNone: false,
-        approvalRequests: false,
-        hostedWebSearch: false,
-        hostedFileSearch: false,
-        remoteMcp: false,
-        computerUse: false,
-        codeExecution: false,
-        toolsets: false
-    }
-};
-const speechCapabilities = {
-    ...transcriptionCapabilities,
-    streaming: true,
-    audioInput: false,
-    audioOutput: true
-};
-const groundedCapabilities = {
-    ...capabilities,
-    webSearch: true
-};
-const imageGenerationCapabilities = {
-    ...capabilities,
-    streaming: false,
-    tools: false,
-    structuredOutput: false,
-    jsonMode: false,
-    toolChoice: false,
-    parallelToolCalls: false,
-    embeddings: false,
-    imageGeneration: true,
-    videoGeneration: false,
-    musicGeneration: false,
-    reasoning: false,
-    webSearch: false,
-    agentCapabilities: {
-        supportTier: "tier-c",
-        toolChoiceNone: false,
-        approvalRequests: false,
-        hostedWebSearch: false,
-        hostedFileSearch: false,
-        remoteMcp: false,
-        computerUse: false,
-        codeExecution: false,
-        toolsets: false
-    }
-};
-const videoGenerationCapabilities = {
-    ...capabilities,
-    streaming: false,
-    tools: false,
-    structuredOutput: false,
-    jsonMode: false,
-    toolChoice: false,
-    parallelToolCalls: false,
-    vision: false,
-    embeddings: false,
-    imageGeneration: false,
-    videoGeneration: true,
-    musicGeneration: false,
-    reasoning: false,
-    webSearch: false,
-    agentCapabilities: {
-        supportTier: "tier-c",
-        toolChoiceNone: false,
-        approvalRequests: false,
-        hostedWebSearch: false,
-        hostedFileSearch: false,
-        remoteMcp: false,
-        computerUse: false,
-        codeExecution: false,
-        toolsets: false
-    }
-};
-const musicGenerationCapabilities = {
-    ...capabilities,
-    streaming: false,
-    tools: false,
-    structuredOutput: false,
-    jsonMode: false,
-    toolChoice: false,
-    parallelToolCalls: false,
-    embeddings: false,
-    imageGeneration: false,
-    videoGeneration: false,
-    musicGeneration: true,
-    reasoning: false,
-    webSearch: false,
-    agentCapabilities: {
-        supportTier: "tier-c",
-        toolChoiceNone: false,
-        approvalRequests: false,
-        hostedWebSearch: false,
-        hostedFileSearch: false,
-        remoteMcp: false,
-        computerUse: false,
-        codeExecution: false,
-        toolsets: false
-    }
-};
-const realtimeCapabilities = (modelId) => {
-    const translation = isGeminiLiveTranslateModel(modelId);
-    return {
-        ...capabilities,
-        streaming: false,
-        tools: !translation,
-        structuredOutput: false,
-        jsonMode: false,
-        toolChoice: false,
-        parallelToolCalls: false,
-        vision: !translation,
-        files: false,
-        audioInput: true,
-        audioOutput: true,
-        embeddings: false,
-        fileSearch: false,
-        urlContext: false,
-        contextCaching: false,
-        batch: false,
-        interactions: false,
-        rawPrediction: false,
-        computerUse: false,
-        reasoning: !translation,
-        webSearch: !translation,
-        agentCapabilities: {
-            ...capabilities.agentCapabilities,
-            toolChoiceNone: !translation,
-            hostedWebSearch: !translation,
-            hostedFileSearch: false,
-            remoteMcp: false,
-            computerUse: false,
-            codeExecution: false
-        },
-        realtime: {
-            sessions: true,
-            audioInput: true,
-            audioOutput: true,
-            imageInput: !translation,
-            tools: !translation,
-            browserTokens: true
-        }
-    };
-};
 const MIB = 1024 * 1024;
 const MAX_JSON_RESPONSE_BYTES = 128 * MIB;
 const MAX_MEDIA_RESPONSE_BYTES = 64 * MIB;
@@ -749,7 +560,6 @@ const mapRealtimeProviderOptions = (providerOptions) => providerOptions
         "translationConfig"
     ].includes(key)))
     : {};
-const isGeminiLiveTranslateModel = (modelId) => /^gemini-3\.5-live-translate(?:-preview)?$/i.test(modelId.trim());
 const isGemini31FlashLiveModel = (modelId) => /^gemini-3\.1-flash-live(?:-preview)?$/i.test(modelId.trim());
 const geminiRealtimeURL = (baseURL, apiKey, providerOptions) => {
     const override = providerOptions?.realtime_url;
@@ -832,8 +642,30 @@ const assertGeminiRealtimeTranslateConfig = (config, modelId) => {
         throw new UnsupportedFeatureError('Model "gemini/gemini-3.5-live-translate-preview" does not support realtime system instructions.');
     }
 };
+const assertGeminiRealtimeTranscribeConfig = (config, modelId) => {
+    if (!isGeminiLiveTranscribeModel(modelId)) {
+        return;
+    }
+    if (config.mode && config.mode !== "transcription") {
+        throw new UnsupportedFeatureError('Model "gemini/gemini-3.5-transcribe-live" only supports realtime transcription mode.');
+    }
+    if (config.voice || config.outputAudioTranscription) {
+        throw new UnsupportedFeatureError('Model "gemini/gemini-3.5-transcribe-live" produces text transcripts and does not support audio output.');
+    }
+    if (config.translation) {
+        throw new UnsupportedFeatureError('Model "gemini/gemini-3.5-transcribe-live" does not support realtime translation.');
+    }
+    const tools = toToolSet(config.tools);
+    if (tools && Object.keys(tools).length > 0) {
+        throw new UnsupportedFeatureError('Model "gemini/gemini-3.5-transcribe-live" does not support realtime tools.');
+    }
+    if (config.reasoning || config.instructions) {
+        throw new UnsupportedFeatureError('Model "gemini/gemini-3.5-transcribe-live" does not support reasoning or system instructions.');
+    }
+};
 const assertGeminiRealtimeConfig = (config, modelId) => {
     assertGeminiRealtimeTranslateConfig(config, modelId);
+    assertGeminiRealtimeTranscribeConfig(config, modelId);
     if (config.toolChoice !== undefined && !["auto", "none"].includes(String(config.toolChoice))) {
         throw new UnsupportedFeatureError("Gemini Live supports automatic tool selection or tool disabling, but not required or named tool choice.");
     }
@@ -851,7 +683,7 @@ const geminiRealtimeSetup = (config, modelId) => ({
     setup: {
         model: `models/${modelId}`,
         generationConfig: {
-            responseModalities: ["AUDIO"],
+            responseModalities: [isGeminiLiveTranscribeModel(modelId) ? "TEXT" : "AUDIO"],
             ...(config.voice
                 ? {
                     speechConfig: {
@@ -866,9 +698,11 @@ const geminiRealtimeSetup = (config, modelId) => ({
             ...(mapRealtimeThinkingConfig(config) ? { thinkingConfig: mapRealtimeThinkingConfig(config) } : {})
         },
         ...(mapRealtimeTranslationConfig(config) ? { translationConfig: mapRealtimeTranslationConfig(config) } : {}),
-        ...(mapRealtimeTranscriptionConfig(config.inputAudioTranscription ?? (config.inputTranscription ? true : undefined))
+        ...(mapRealtimeTranscriptionConfig(config.inputAudioTranscription ??
+            (config.inputTranscription || isGeminiLiveTranscribeModel(modelId) ? true : undefined))
             ? {
-                inputAudioTranscription: mapRealtimeTranscriptionConfig(config.inputAudioTranscription ?? (config.inputTranscription ? true : undefined))
+                inputAudioTranscription: mapRealtimeTranscriptionConfig(config.inputAudioTranscription ??
+                    (config.inputTranscription || isGeminiLiveTranscribeModel(modelId) ? true : undefined))
             }
             : {}),
         ...(mapRealtimeTranscriptionConfig(config.outputAudioTranscription)
@@ -1097,17 +931,23 @@ const createGeminiRealtimeEventParser = () => {
 };
 const isGemini3Model = (modelId) => /^gemini-3([.-]|$)/.test(modelId);
 const isGemini3ProModel = (modelId) => /^gemini-3([.-].*)?pro([.-]|$)/.test(modelId);
-const usesCurrentGeminiRequestRules = (modelId) => modelId === "gemini-3.6-flash" || modelId === "gemini-3.5-flash-lite";
+const usesCurrentGeminiRequestRules = (modelId) => modelId === "gemini-3.8-flash" ||
+    modelId === "gemini-3.7-flash" ||
+    modelId === "gemini-3.6-flash" ||
+    modelId === "gemini-3.5-flash-lite";
 const currentGeminiReasoningEfforts = [
     "minimal",
     "low",
     "medium",
     "high"
 ];
+const reasoningEffortsForModel = (modelId) => (modelId === "gemini-3.8-flash" || modelId === "gemini-3.7-flash")
+    ? ["low", "medium", "high"]
+    : currentGeminiReasoningEfforts;
 const modelCapabilities = (modelId, baseCapabilities = capabilities) => usesCurrentGeminiRequestRules(modelId)
     ? {
         ...baseCapabilities,
-        reasoningEfforts: [...currentGeminiReasoningEfforts]
+        reasoningEfforts: [...reasoningEffortsForModel(modelId)]
     }
     : baseCapabilities;
 const currentGeminiGenerationControlKeys = [
@@ -1144,7 +984,23 @@ const assertCurrentGeminiGenerateInput = (provider, modelId, input) => {
     if (!usesCurrentGeminiRequestRules(modelId)) {
         return;
     }
+    if (input.reasoning?.effort !== undefined && !reasoningEffortsForModel(modelId).includes(input.reasoning.effort)) {
+        throw new UnsupportedFeatureError(`Provider "${provider}" does not support reasoning effort "${input.reasoning.effort}" for model "${modelId}".`);
+    }
     const providerOptions = input.providerOptions;
+    const rawConfig = providerOptions?.generationConfig ?? providerOptions?.generation_config;
+    const config = rawConfig && typeof rawConfig === "object" ? rawConfig : {};
+    const rawThinking = config.thinkingConfig ?? config.thinking_config ?? providerOptions?.thinkingConfig ?? providerOptions?.thinking_config;
+    if (rawThinking && typeof rawThinking === "object") {
+        const thinking = rawThinking;
+        const effort = thinking.thinkingLevel ?? thinking.thinking_level;
+        if (effort !== undefined && !reasoningEffortsForModel(modelId).some((supported) => supported === String(effort).toLowerCase())) {
+            throw new UnsupportedFeatureError(`Provider "${provider}" does not support thinking level "${effort}" for model "${modelId}".`);
+        }
+        if (thinking.thinkingBudget !== undefined || thinking.thinking_budget !== undefined) {
+            throw new UnsupportedFeatureError(`Provider "${provider}" requires thinking levels instead of budgets for model "${modelId}".`);
+        }
+    }
     const unsupportedControl = firstUnsupportedGenerationControl(input.temperature === undefined ? undefined : { temperature: input.temperature }, providerOptions, providerOptions?.generationConfig, providerOptions?.generation_config);
     if (unsupportedControl) {
         throw new UnsupportedFeatureError(`Provider "${provider}" does not support generation control "${unsupportedControl}" for model "${modelId}". ` +
@@ -2194,15 +2050,84 @@ class GeminiTranscriptionModel {
     apiKey;
     baseURL;
     fetcher;
+    allowUnsafeEndpoints;
     provider = "gemini";
     capabilities = transcriptionCapabilities;
-    constructor(modelId, apiKey, baseURL, fetcher) {
+    constructor(modelId, apiKey, baseURL, fetcher, allowUnsafeEndpoints = false) {
         this.modelId = modelId;
         this.apiKey = apiKey;
         this.baseURL = baseURL;
         this.fetcher = fetcher;
+        this.allowUnsafeEndpoints = allowUnsafeEndpoints;
+    }
+    isDedicatedTranscriptionModel() {
+        return /^gemini-3\.5-transcribe$/i.test(this.modelId.trim());
+    }
+    async transcribeDedicated(input) {
+        if (input.prompt) {
+            throw new UnsupportedFeatureError('Model "gemini/gemini-3.5-transcribe" does not support a free-form transcription prompt. Use providerOptions.custom_vocabulary for terminology hints.');
+        }
+        const audioBytes = typeof input.audio.data === "string"
+            ? Buffer.from(input.audio.data, "base64")
+            : input.audio.data instanceof Uint8Array
+                ? input.audio.data
+                : new Uint8Array(input.audio.data);
+        const requestOptions = {
+            abortSignal: input.abortSignal,
+            timeoutMs: input.timeoutMs,
+            maxRetries: input.maxRetries,
+            retryBackoffMs: input.retryBackoffMs
+        };
+        const files = new GeminiFilesClient(this.apiKey, this.baseURL, this.fetcher, this.allowUnsafeEndpoints);
+        const uploaded = await files.upload({
+            ...requestOptions,
+            data: audioBytes,
+            mediaType: input.audio.mediaType,
+            displayName: input.audio.filename
+        });
+        if (!uploaded.name) {
+            throw new ProviderHTTPError('Gemini Files API did not return a name for the transcription upload.', 500);
+        }
+        try {
+            if (!uploaded.uri) {
+                throw new ProviderHTTPError('Gemini Files API did not return a URI for the transcription upload.', 500);
+            }
+            const interaction = await new GeminiInteractionsClient(this.apiKey, this.baseURL, this.fetcher).create({
+                ...requestOptions,
+                modelId: this.modelId,
+                input: [
+                    {
+                        type: "audio",
+                        uri: uploaded.uri,
+                        mime_type: uploaded.mimeType ?? input.audio.mediaType
+                    }
+                ],
+                generationConfig: {
+                    transcription_config: {
+                        ...(input.providerOptions ?? {}),
+                        ...(input.language ? { language_codes: [input.language] } : {})
+                    }
+                },
+                store: false
+            });
+            return {
+                text: interaction.outputText ?? "",
+                rawResponse: interaction.rawResponse
+            };
+        }
+        finally {
+            await files.delete({
+                name: uploaded.name,
+                timeoutMs: input.timeoutMs,
+                maxRetries: input.maxRetries,
+                retryBackoffMs: input.retryBackoffMs
+            });
+        }
     }
     async transcribe(input) {
+        if (this.isDedicatedTranscriptionModel()) {
+            return this.transcribeDedicated(input);
+        }
         const { signal, cleanup } = withTimeoutSignal(input);
         try {
             const response = await withRetry(() => this.fetcher(`${this.baseURL}/models/${encodeGeminiPathSegment(this.modelId, "Gemini model ID")}:generateContent?key=${this.apiKey}`, {
@@ -2622,8 +2547,8 @@ class GeminiRealtimeModel {
                     }
                 ],
                 buildMediaPayloads: (frame) => {
-                    if (isGeminiLiveTranslateModel(this.modelId)) {
-                        throw new UnsupportedFeatureError('Model "gemini/gemini-3.5-live-translate-preview" only supports audio input.');
+                    if (isGeminiLiveTranslateModel(this.modelId) || isGeminiLiveTranscribeModel(this.modelId)) {
+                        throw new UnsupportedFeatureError(`Model "gemini/${this.modelId}" only supports audio input.`);
                     }
                     return [
                         {
@@ -2637,8 +2562,8 @@ class GeminiRealtimeModel {
                     ];
                 },
                 buildTextPayloads: (text) => {
-                    if (isGeminiLiveTranslateModel(this.modelId)) {
-                        throw new UnsupportedFeatureError('Model "gemini/gemini-3.5-live-translate-preview" only supports audio input.');
+                    if (isGeminiLiveTranslateModel(this.modelId) || isGeminiLiveTranscribeModel(this.modelId)) {
+                        throw new UnsupportedFeatureError(`Model "gemini/${this.modelId}" only supports audio input.`);
                     }
                     return [
                         {
@@ -2761,7 +2686,7 @@ export const createGemini = (options = {}) => {
         name: "gemini",
         languageModel: (modelId) => new GeminiLanguageModel(modelId, apiKey, baseURL, fetcher),
         embeddingModel: (modelId) => new GeminiEmbeddingModel(modelId, apiKey, baseURL, fetcher),
-        transcriptionModel: (modelId) => new GeminiTranscriptionModel(modelId, apiKey, baseURL, fetcher),
+        transcriptionModel: (modelId) => new GeminiTranscriptionModel(modelId, apiKey, baseURL, fetcher, options.allowUnsafeEndpoints),
         speechModel: (modelId) => new GeminiSpeechModel(modelId, apiKey, baseURL, fetcher),
         imageGenerationModel: (modelId) => new GeminiImageGenerationModel(modelId, apiKey, baseURL, fetcher),
         videoGenerationModel: (modelId) => new GeminiVideoGenerationModel(modelId, apiKey, baseURL, fetcher),