@aigne/gemini 0.14.2-beta → 0.14.2-beta.10

package/CHANGELOG.md

@@ -1,5 +1,139 @@
 # Changelog
 
+## [0.14.2-beta.10](https://github.com/AIGNE-io/aigne-framework/compare/gemini-v0.14.2-beta.9...gemini-v0.14.2-beta.10) (2025-10-16)
+
+
+### Bug Fixes
+
+* correct calculate token usage for gemini model ([7fd1328](https://github.com/AIGNE-io/aigne-framework/commit/7fd13289d3d0f8e062211f7c6dd5cb56e5318c1b))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/core bumped to 1.63.0-beta.10
+  * devDependencies
+    * @aigne/test-utils bumped to 0.5.55-beta.10
+
+## [0.14.2-beta.9](https://github.com/AIGNE-io/aigne-framework/compare/gemini-v0.14.2-beta.8...gemini-v0.14.2-beta.9) (2025-10-16)
+
+
+### Bug Fixes
+
+* **models:** auto retry when got emtpy response from gemini ([#636](https://github.com/AIGNE-io/aigne-framework/issues/636)) ([9367cef](https://github.com/AIGNE-io/aigne-framework/commit/9367cef49ea4c0c87b8a36b454deb2efaee6886f))
+* **models:** enhance gemini model tool use with status fields ([#634](https://github.com/AIGNE-io/aigne-framework/issues/634)) ([067b175](https://github.com/AIGNE-io/aigne-framework/commit/067b175c8e31bb5b1a6d0fc5a5cfb2d070d8d709))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/core bumped to 1.63.0-beta.9
+  * devDependencies
+    * @aigne/test-utils bumped to 0.5.55-beta.9
+
+## [0.14.2-beta.8](https://github.com/AIGNE-io/aigne-framework/compare/gemini-v0.14.2-beta.7...gemini-v0.14.2-beta.8) (2025-10-16)
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/openai bumped to 0.16.2-beta.8
+  * devDependencies
+    * @aigne/core bumped to 1.63.0-beta.8
+    * @aigne/test-utils bumped to 0.5.55-beta.8
+
+## [0.14.2-beta.7](https://github.com/AIGNE-io/aigne-framework/compare/gemini-v0.14.2-beta.6...gemini-v0.14.2-beta.7) (2025-10-15)
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/openai bumped to 0.16.2-beta.7
+  * devDependencies
+    * @aigne/core bumped to 1.63.0-beta.7
+    * @aigne/test-utils bumped to 0.5.55-beta.7
+
+## [0.14.2-beta.6](https://github.com/AIGNE-io/aigne-framework/compare/gemini-v0.14.2-beta.5...gemini-v0.14.2-beta.6) (2025-10-15)
+
+
+### Bug Fixes
+
+* **models:** add imageConfig to gemini image model ([#621](https://github.com/AIGNE-io/aigne-framework/issues/621)) ([252de7a](https://github.com/AIGNE-io/aigne-framework/commit/252de7a10701c4f5302c2fff977c88e5e833b7b1))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/openai bumped to 0.16.2-beta.6
+  * devDependencies
+    * @aigne/core bumped to 1.63.0-beta.6
+    * @aigne/test-utils bumped to 0.5.55-beta.6
+
+## [0.14.2-beta.5](https://github.com/AIGNE-io/aigne-framework/compare/gemini-v0.14.2-beta.4...gemini-v0.14.2-beta.5) (2025-10-13)
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/openai bumped to 0.16.2-beta.5
+  * devDependencies
+    * @aigne/core bumped to 1.63.0-beta.5
+    * @aigne/test-utils bumped to 0.5.55-beta.5
+
+## [0.14.2-beta.4](https://github.com/AIGNE-io/aigne-framework/compare/gemini-v0.14.2-beta.3...gemini-v0.14.2-beta.4) (2025-10-12)
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/openai bumped to 0.16.2-beta.4
+  * devDependencies
+    * @aigne/core bumped to 1.63.0-beta.4
+    * @aigne/test-utils bumped to 0.5.55-beta.4
+
+## [0.14.2-beta.3](https://github.com/AIGNE-io/aigne-framework/compare/gemini-v0.14.2-beta.2...gemini-v0.14.2-beta.3) (2025-10-11)
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/openai bumped to 0.16.2-beta.3
+  * devDependencies
+    * @aigne/core bumped to 1.63.0-beta.3
+    * @aigne/test-utils bumped to 0.5.55-beta.3
+
+## [0.14.2-beta.2](https://github.com/AIGNE-io/aigne-framework/compare/gemini-v0.14.2-beta.1...gemini-v0.14.2-beta.2) (2025-10-09)
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/openai bumped to 0.16.2-beta.2
+  * devDependencies
+    * @aigne/core bumped to 1.63.0-beta.2
+    * @aigne/test-utils bumped to 0.5.55-beta.2
+
+## [0.14.2-beta.1](https://github.com/AIGNE-io/aigne-framework/compare/gemini-v0.14.2-beta...gemini-v0.14.2-beta.1) (2025-10-09)
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @aigne/openai bumped to 0.16.2-beta.1
+  * devDependencies
+    * @aigne/core bumped to 1.63.0-beta.1
+    * @aigne/test-utils bumped to 0.5.55-beta.1
+
 ## [0.14.2-beta](https://github.com/AIGNE-io/aigne-framework/compare/gemini-v0.14.1...gemini-v0.14.2-beta) (2025-10-07)
 
 

package/lib/cjs/gemini-chat-model.d.ts

@@ -1,13 +1,21 @@
-import { type AgentInvokeOptions, type AgentProcessResult, type ChatModelInput, type ChatModelOutput } from "@aigne/core";
+import { type AgentProcessResult, ChatModel, type ChatModelInput, type ChatModelOptions, type ChatModelOutput } from "@aigne/core";
 import { type PromiseOrValue } from "@aigne/core/utils/type-utils.js";
-import { OpenAIChatModel, type OpenAIChatModelOptions } from "@aigne/openai";
-import { GoogleGenAI } from "@google/genai";
+import { GoogleGenAI, type GoogleGenAIOptions } from "@google/genai";
+export interface GeminiChatModelOptions extends ChatModelOptions {
+    /**
+     * API key for Gemini API
+     *
+     * If not provided, will look for GEMINI_API_KEY or GOOGLE_API_KEY in environment variables
+     */
+    apiKey?: string;
+    /**
+     * Optional client options for the Gemini SDK
+     */
+    clientOptions?: Partial<GoogleGenAIOptions>;
+}
 /**
  * Implementation of the ChatModel interface for Google's Gemini API
  *
- * This model uses OpenAI-compatible API format to interact with Google's Gemini models,
- * providing access to models like Gemini 1.5 and Gemini 2.0.
- *
  * @example
  * Here's how to create and use a Gemini chat model:
  * {@includeCode ../test/gemini-chat-model.test.ts#example-gemini-chat-model}
@@ -16,19 +24,20 @@ import { GoogleGenAI } from "@google/genai";
  * Here's an example with streaming response:
  * {@includeCode ../test/gemini-chat-model.test.ts#example-gemini-chat-model-streaming}
  */
-export declare class GeminiChatModel extends OpenAIChatModel {
-    constructor(options?: OpenAIChatModelOptions);
+export declare class GeminiChatModel extends ChatModel {
+    options?: GeminiChatModelOptions | undefined;
+    constructor(options?: GeminiChatModelOptions | undefined);
     protected apiKeyEnvName: string;
-    protected supportsToolsUseWithJsonSchema: boolean;
-    protected supportsParallelToolCalls: boolean;
-    protected supportsToolStreaming: boolean;
-    protected optionalFieldMode: "optional";
     protected _googleClient?: GoogleGenAI;
     get googleClient(): GoogleGenAI;
-    process(input: ChatModelInput, options: AgentInvokeOptions): PromiseOrValue<AgentProcessResult<ChatModelOutput>>;
-    private handleImageModelProcessing;
+    get credential(): {
+        apiKey: string | undefined;
+        model: string;
+    };
+    get modelOptions(): Omit<import("@aigne/core").ChatModelInputOptions, "model"> | undefined;
+    process(input: ChatModelInput): PromiseOrValue<AgentProcessResult<ChatModelOutput>>;
+    private processInput;
     private buildConfig;
     private buildTools;
     private buildContents;
-    getRunMessages(input: ChatModelInput): ReturnType<OpenAIChatModel["getRunMessages"]>;
 }

package/lib/cjs/gemini-chat-model.js

@@ -2,18 +2,15 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GeminiChatModel = void 0;
 const core_1 = require("@aigne/core");
+const logger_js_1 = require("@aigne/core/utils/logger.js");
 const type_utils_js_1 = require("@aigne/core/utils/type-utils.js");
-const openai_1 = require("@aigne/openai");
 const uuid_1 = require("@aigne/uuid");
 const genai_1 = require("@google/genai");
-const GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai";
 const GEMINI_DEFAULT_CHAT_MODEL = "gemini-2.0-flash";
+const OUTPUT_JSON_FUNCTION_NAME = "output_json";
 /**
  * Implementation of the ChatModel interface for Google's Gemini API
  *
- * This model uses OpenAI-compatible API format to interact with Google's Gemini models,
- * providing access to models like Gemini 1.5 and Gemini 2.0.
- *
  * @example
  * Here's how to create and use a Gemini chat model:
  * {@includeCode ../test/gemini-chat-model.test.ts#example-gemini-chat-model}
@@ -22,19 +19,16 @@ const GEMINI_DEFAULT_CHAT_MODEL = "gemini-2.0-flash";
  * Here's an example with streaming response:
  * {@includeCode ../test/gemini-chat-model.test.ts#example-gemini-chat-model-streaming}
  */
-class GeminiChatModel extends openai_1.OpenAIChatModel {
+class GeminiChatModel extends core_1.ChatModel {
+    options;
     constructor(options) {
         super({
             ...options,
             model: options?.model || GEMINI_DEFAULT_CHAT_MODEL,
-            baseURL: options?.baseURL || GEMINI_BASE_URL,
         });
+        this.options = options;
     }
     apiKeyEnvName = "GEMINI_API_KEY";
-    supportsToolsUseWithJsonSchema = false;
-    supportsParallelToolCalls = false;
-    supportsToolStreaming = false;
-    optionalFieldMode = "optional";
     _googleClient;
     get googleClient() {
         if (this._googleClient)
@@ -42,20 +36,33 @@ class GeminiChatModel extends openai_1.OpenAIChatModel {
         const { apiKey } = this.credential;
         if (!apiKey)
             throw new Error(`${this.name} requires an API key. Please provide it via \`options.apiKey\`, or set the \`${this.apiKeyEnvName}\` environment variable`);
-        this._googleClient ??= new genai_1.GoogleGenAI({ apiKey });
+        this._googleClient ??= new genai_1.GoogleGenAI({
+            apiKey,
+            ...this.options?.clientOptions,
+        });
         return this._googleClient;
     }
-    process(input, options) {
-        const model = input.modelOptions?.model || this.credential.model;
-        if (!model.includes("image"))
-            return super.process(input, options);
-        return this.handleImageModelProcessing(input);
+    get credential() {
+        const apiKey = this.options?.apiKey ||
+            process.env[this.apiKeyEnvName] ||
+            process.env.GEMINI_API_KEY ||
+            process.env.GOOGLE_API_KEY;
+        return {
+            apiKey,
+            model: this.options?.model || GEMINI_DEFAULT_CHAT_MODEL,
+        };
     }
-    async *handleImageModelProcessing(input) {
+    get modelOptions() {
+        return this.options?.modelOptions;
+    }
+    process(input) {
+        return this.processInput(input);
+    }
+    async *processInput(input) {
         const model = input.modelOptions?.model || this.credential.model;
         const { contents, config } = await this.buildContents(input);
         const parameters = {
-            model: model,
+            model,
             contents,
             config: {
                 responseModalities: input.modelOptions?.modalities,
@@ -64,7 +71,6 @@ class GeminiChatModel extends openai_1.OpenAIChatModel {
                 frequencyPenalty: input.modelOptions?.frequencyPenalty || this.modelOptions?.frequencyPenalty,
                 presencePenalty: input.modelOptions?.presencePenalty || this.modelOptions?.presencePenalty,
                 ...config,
-                ...(await this.buildTools(input)),
                 ...(await this.buildConfig(input)),
             },
         };
@@ -77,6 +83,7 @@ class GeminiChatModel extends openai_1.OpenAIChatModel {
         const files = [];
         const toolCalls = [];
         let text = "";
+        let json;
         for await (const chunk of response) {
             if (!responseModel && chunk.modelVersion) {
                 responseModel = chunk.modelVersion;
@@ -100,34 +107,77 @@ class GeminiChatModel extends openai_1.OpenAIChatModel {
                             });
                         }
                         if (part.functionCall?.name) {
-                            toolCalls.push({
-                                id: part.functionCall.id || (0, uuid_1.v7)(),
-                                type: "function",
-                                function: {
-                                    name: part.functionCall.name,
-                                    arguments: part.functionCall.args || {},
-                                },
-                            });
-                            yield { delta: { json: { toolCalls } } };
+                            if (part.functionCall.name === OUTPUT_JSON_FUNCTION_NAME) {
+                                json = part.functionCall.args;
+                            }
+                            else {
+                                toolCalls.push({
+                                    id: part.functionCall.id || (0, uuid_1.v7)(),
+                                    type: "function",
+                                    function: {
+                                        name: part.functionCall.name,
+                                        arguments: part.functionCall.args || {},
+                                    },
+                                });
+                                yield { delta: { json: { toolCalls } } };
+                            }
                         }
                     }
                 }
             }
             if (chunk.usageMetadata) {
-                usage.inputTokens += chunk.usageMetadata.promptTokenCount || 0;
-                usage.outputTokens += chunk.usageMetadata.candidatesTokenCount || 0;
+                if (chunk.usageMetadata.promptTokenCount)
+                    usage.inputTokens = chunk.usageMetadata.promptTokenCount;
+                if (chunk.usageMetadata.candidatesTokenCount)
+                    usage.outputTokens = chunk.usageMetadata.candidatesTokenCount;
             }
         }
         if (input.responseFormat?.type === "json_schema") {
-            yield { delta: { json: { json: (0, core_1.safeParseJSON)(text) } } };
+            if (json) {
+                yield { delta: { json: { json } } };
+            }
+            else if (text) {
+                yield { delta: { json: { json: (0, core_1.safeParseJSON)(text) } } };
+            }
+            else {
+                // NOTE: Trigger retry of chat model
+                throw new core_1.StructuredOutputError("No JSON response from the model");
+            }
+        }
+        else if (!toolCalls.length) {
+            if (!text) {
+                logger_js_1.logger.error("No text response from the model", parameters);
+                // NOTE: Trigger retry of chat model
+                throw new core_1.StructuredOutputError("No text response from the model");
+            }
         }
-        yield { delta: { json: { usage, files } } };
+        yield { delta: { json: { usage, files: files.length ? files : undefined } } };
     }
     async buildConfig(input) {
         const config = {};
+        const { tools, toolConfig } = await this.buildTools(input);
+        config.tools = tools;
+        config.toolConfig = toolConfig;
         if (input.responseFormat?.type === "json_schema") {
-            config.responseJsonSchema = input.responseFormat.jsonSchema.schema;
-            config.responseMimeType = "application/json";
+            if (config.tools?.length) {
+                config.tools.push({
+                    functionDeclarations: [
+                        {
+                            name: OUTPUT_JSON_FUNCTION_NAME,
+                            description: "Output the final response in JSON format",
+                            parametersJsonSchema: input.responseFormat.jsonSchema.schema,
+                        },
+                    ],
+                });
+                config.toolConfig = {
+                    ...config.toolConfig,
+                    functionCallingConfig: { mode: genai_1.FunctionCallingConfigMode.ANY },
+                };
+            }
+            else {
+                config.responseJsonSchema = input.responseFormat.jsonSchema.schema;
+                config.responseMimeType = "application/json";
+            }
         }
         return config;
     }
@@ -178,7 +228,7 @@ class GeminiChatModel extends openai_1.OpenAIChatModel {
                 return;
             }
             const content = {
-                role: msg.role === "agent" ? "model" : "user",
+                role: msg.role === "agent" ? "model" : msg.role === "user" ? "user" : undefined,
             };
             if (msg.toolCalls) {
                 content.parts = msg.toolCalls.map((call) => ({
@@ -195,12 +245,31 @@ class GeminiChatModel extends openai_1.OpenAIChatModel {
                     .find((c) => c?.id === msg.toolCallId);
                 if (!call)
                     throw new Error(`Tool call not found: ${msg.toolCallId}`);
+                const output = JSON.parse(msg.content);
+                const isError = "error" in output && Boolean(input.error);
+                const response = {
+                    tool: call.function.name,
+                };
+                // NOTE: base on the documentation of gemini api, the content should include `output` field for successful result or `error` field for failed result,
+                // and base on the actual test, add a tool field presenting the tool name can improve the LLM understanding that which tool is called.
+                if (isError) {
+                    Object.assign(response, { status: "error" }, output);
+                }
+                else {
+                    Object.assign(response, { status: "success" });
+                    if ("output" in output) {
+                        Object.assign(response, output);
+                    }
+                    else {
+                        Object.assign(response, { output });
+                    }
+                }
                 content.parts = [
                     {
                         functionResponse: {
                             id: msg.toolCallId,
                             name: call.function.name,
-                            response: JSON.parse(msg.content),
+                            response,
                         },
                     },
                 ];
@@ -224,24 +293,17 @@ class GeminiChatModel extends openai_1.OpenAIChatModel {
             }
             return content;
         }))).filter(type_utils_js_1.isNonNullable);
+        if (!result.contents.length && systemParts.length) {
+            const system = systemParts.pop();
+            if (system) {
+                result.contents.push({ role: "user", parts: [system] });
+            }
+        }
         if (systemParts.length) {
             result.config ??= {};
             result.config.systemInstruction = systemParts;
         }
         return result;
     }
-    async getRunMessages(input) {
-        const messages = await super.getRunMessages(input);
-        if (!messages.some((i) => i.role === "user")) {
-            for (const msg of messages) {
-                if (msg.role === "system") {
-                    // Ensure the last message is from the user
-                    msg.role = "user";
-                    break;
-                }
-            }
-        }
-        return messages;
-    }
 }
 exports.GeminiChatModel = GeminiChatModel;

package/lib/cjs/gemini-image-model.js

@@ -132,6 +132,7 @@ class GeminiImageModel extends core_1.ImageModel {
             "tools",
             "topK",
             "topP",
+            "imageConfig",
         ];
         const images = await Promise.all((0, type_utils_js_1.flat)(input.image).map(async (image) => {
             const { data, mimeType } = await this.transformFileType("file", image, options);

package/lib/dts/gemini-chat-model.d.ts

@@ -1,13 +1,21 @@
-import { type AgentInvokeOptions, type AgentProcessResult, type ChatModelInput, type ChatModelOutput } from "@aigne/core";
+import { type AgentProcessResult, ChatModel, type ChatModelInput, type ChatModelOptions, type ChatModelOutput } from "@aigne/core";
 import { type PromiseOrValue } from "@aigne/core/utils/type-utils.js";
-import { OpenAIChatModel, type OpenAIChatModelOptions } from "@aigne/openai";
-import { GoogleGenAI } from "@google/genai";
+import { GoogleGenAI, type GoogleGenAIOptions } from "@google/genai";
+export interface GeminiChatModelOptions extends ChatModelOptions {
+    /**
+     * API key for Gemini API
+     *
+     * If not provided, will look for GEMINI_API_KEY or GOOGLE_API_KEY in environment variables
+     */
+    apiKey?: string;
+    /**
+     * Optional client options for the Gemini SDK
+     */
+    clientOptions?: Partial<GoogleGenAIOptions>;
+}
 /**
  * Implementation of the ChatModel interface for Google's Gemini API
  *
- * This model uses OpenAI-compatible API format to interact with Google's Gemini models,
- * providing access to models like Gemini 1.5 and Gemini 2.0.
- *
  * @example
  * Here's how to create and use a Gemini chat model:
  * {@includeCode ../test/gemini-chat-model.test.ts#example-gemini-chat-model}
@@ -16,19 +24,20 @@ import { GoogleGenAI } from "@google/genai";
  * Here's an example with streaming response:
  * {@includeCode ../test/gemini-chat-model.test.ts#example-gemini-chat-model-streaming}
  */
-export declare class GeminiChatModel extends OpenAIChatModel {
-    constructor(options?: OpenAIChatModelOptions);
+export declare class GeminiChatModel extends ChatModel {
+    options?: GeminiChatModelOptions | undefined;
+    constructor(options?: GeminiChatModelOptions | undefined);
     protected apiKeyEnvName: string;
-    protected supportsToolsUseWithJsonSchema: boolean;
-    protected supportsParallelToolCalls: boolean;
-    protected supportsToolStreaming: boolean;
-    protected optionalFieldMode: "optional";
     protected _googleClient?: GoogleGenAI;
     get googleClient(): GoogleGenAI;
-    process(input: ChatModelInput, options: AgentInvokeOptions): PromiseOrValue<AgentProcessResult<ChatModelOutput>>;
-    private handleImageModelProcessing;
+    get credential(): {
+        apiKey: string | undefined;
+        model: string;
+    };
+    get modelOptions(): Omit<import("@aigne/core").ChatModelInputOptions, "model"> | undefined;
+    process(input: ChatModelInput): PromiseOrValue<AgentProcessResult<ChatModelOutput>>;
+    private processInput;
     private buildConfig;
     private buildTools;
     private buildContents;
-    getRunMessages(input: ChatModelInput): ReturnType<OpenAIChatModel["getRunMessages"]>;
 }

package/lib/esm/gemini-chat-model.d.ts

@@ -1,13 +1,21 @@
-import { type AgentInvokeOptions, type AgentProcessResult, type ChatModelInput, type ChatModelOutput } from "@aigne/core";
+import { type AgentProcessResult, ChatModel, type ChatModelInput, type ChatModelOptions, type ChatModelOutput } from "@aigne/core";
 import { type PromiseOrValue } from "@aigne/core/utils/type-utils.js";
-import { OpenAIChatModel, type OpenAIChatModelOptions } from "@aigne/openai";
-import { GoogleGenAI } from "@google/genai";
+import { GoogleGenAI, type GoogleGenAIOptions } from "@google/genai";
+export interface GeminiChatModelOptions extends ChatModelOptions {
+    /**
+     * API key for Gemini API
+     *
+     * If not provided, will look for GEMINI_API_KEY or GOOGLE_API_KEY in environment variables
+     */
+    apiKey?: string;
+    /**
+     * Optional client options for the Gemini SDK
+     */
+    clientOptions?: Partial<GoogleGenAIOptions>;
+}
 /**
  * Implementation of the ChatModel interface for Google's Gemini API
  *
- * This model uses OpenAI-compatible API format to interact with Google's Gemini models,
- * providing access to models like Gemini 1.5 and Gemini 2.0.
- *
  * @example
  * Here's how to create and use a Gemini chat model:
  * {@includeCode ../test/gemini-chat-model.test.ts#example-gemini-chat-model}
@@ -16,19 +24,20 @@ import { GoogleGenAI } from "@google/genai";
  * Here's an example with streaming response:
  * {@includeCode ../test/gemini-chat-model.test.ts#example-gemini-chat-model-streaming}
  */
-export declare class GeminiChatModel extends OpenAIChatModel {
-    constructor(options?: OpenAIChatModelOptions);
+export declare class GeminiChatModel extends ChatModel {
+    options?: GeminiChatModelOptions | undefined;
+    constructor(options?: GeminiChatModelOptions | undefined);
     protected apiKeyEnvName: string;
-    protected supportsToolsUseWithJsonSchema: boolean;
-    protected supportsParallelToolCalls: boolean;
-    protected supportsToolStreaming: boolean;
-    protected optionalFieldMode: "optional";
     protected _googleClient?: GoogleGenAI;
     get googleClient(): GoogleGenAI;
-    process(input: ChatModelInput, options: AgentInvokeOptions): PromiseOrValue<AgentProcessResult<ChatModelOutput>>;
-    private handleImageModelProcessing;
+    get credential(): {
+        apiKey: string | undefined;
+        model: string;
+    };
+    get modelOptions(): Omit<import("@aigne/core").ChatModelInputOptions, "model"> | undefined;
+    process(input: ChatModelInput): PromiseOrValue<AgentProcessResult<ChatModelOutput>>;
+    private processInput;
     private buildConfig;
     private buildTools;
     private buildContents;
-    getRunMessages(input: ChatModelInput): ReturnType<OpenAIChatModel["getRunMessages"]>;
 }

package/lib/esm/gemini-chat-model.js

@@ -1,16 +1,13 @@
-import { safeParseJSON, } from "@aigne/core";
+import { ChatModel, StructuredOutputError, safeParseJSON, } from "@aigne/core";
+import { logger } from "@aigne/core/utils/logger.js";
 import { isNonNullable } from "@aigne/core/utils/type-utils.js";
-import { OpenAIChatModel } from "@aigne/openai";
 import { v7 } from "@aigne/uuid";
 import { FunctionCallingConfigMode, GoogleGenAI, } from "@google/genai";
-const GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai";
 const GEMINI_DEFAULT_CHAT_MODEL = "gemini-2.0-flash";
+const OUTPUT_JSON_FUNCTION_NAME = "output_json";
 /**
  * Implementation of the ChatModel interface for Google's Gemini API
  *
- * This model uses OpenAI-compatible API format to interact with Google's Gemini models,
- * providing access to models like Gemini 1.5 and Gemini 2.0.
- *
  * @example
  * Here's how to create and use a Gemini chat model:
  * {@includeCode ../test/gemini-chat-model.test.ts#example-gemini-chat-model}
@@ -19,19 +16,16 @@ const GEMINI_DEFAULT_CHAT_MODEL = "gemini-2.0-flash";
  * Here's an example with streaming response:
  * {@includeCode ../test/gemini-chat-model.test.ts#example-gemini-chat-model-streaming}
  */
-export class GeminiChatModel extends OpenAIChatModel {
+export class GeminiChatModel extends ChatModel {
+    options;
     constructor(options) {
         super({
             ...options,
             model: options?.model || GEMINI_DEFAULT_CHAT_MODEL,
-            baseURL: options?.baseURL || GEMINI_BASE_URL,
         });
+        this.options = options;
     }
     apiKeyEnvName = "GEMINI_API_KEY";
-    supportsToolsUseWithJsonSchema = false;
-    supportsParallelToolCalls = false;
-    supportsToolStreaming = false;
-    optionalFieldMode = "optional";
     _googleClient;
     get googleClient() {
         if (this._googleClient)
@@ -39,20 +33,33 @@ export class GeminiChatModel extends OpenAIChatModel {
         const { apiKey } = this.credential;
         if (!apiKey)
             throw new Error(`${this.name} requires an API key. Please provide it via \`options.apiKey\`, or set the \`${this.apiKeyEnvName}\` environment variable`);
-        this._googleClient ??= new GoogleGenAI({ apiKey });
+        this._googleClient ??= new GoogleGenAI({
+            apiKey,
+            ...this.options?.clientOptions,
+        });
         return this._googleClient;
     }
-    process(input, options) {
-        const model = input.modelOptions?.model || this.credential.model;
-        if (!model.includes("image"))
-            return super.process(input, options);
-        return this.handleImageModelProcessing(input);
+    get credential() {
+        const apiKey = this.options?.apiKey ||
+            process.env[this.apiKeyEnvName] ||
+            process.env.GEMINI_API_KEY ||
+            process.env.GOOGLE_API_KEY;
+        return {
+            apiKey,
+            model: this.options?.model || GEMINI_DEFAULT_CHAT_MODEL,
+        };
     }
-    async *handleImageModelProcessing(input) {
+    get modelOptions() {
+        return this.options?.modelOptions;
+    }
+    process(input) {
+        return this.processInput(input);
+    }
+    async *processInput(input) {
         const model = input.modelOptions?.model || this.credential.model;
         const { contents, config } = await this.buildContents(input);
         const parameters = {
-            model: model,
+            model,
             contents,
             config: {
                 responseModalities: input.modelOptions?.modalities,
@@ -61,7 +68,6 @@ export class GeminiChatModel extends OpenAIChatModel {
                 frequencyPenalty: input.modelOptions?.frequencyPenalty || this.modelOptions?.frequencyPenalty,
                 presencePenalty: input.modelOptions?.presencePenalty || this.modelOptions?.presencePenalty,
                 ...config,
-                ...(await this.buildTools(input)),
                 ...(await this.buildConfig(input)),
             },
         };
@@ -74,6 +80,7 @@ export class GeminiChatModel extends OpenAIChatModel {
         const files = [];
         const toolCalls = [];
         let text = "";
+        let json;
         for await (const chunk of response) {
             if (!responseModel && chunk.modelVersion) {
                 responseModel = chunk.modelVersion;
@@ -97,34 +104,77 @@ export class GeminiChatModel extends OpenAIChatModel {
                             });
                         }
                         if (part.functionCall?.name) {
-                            toolCalls.push({
-                                id: part.functionCall.id || v7(),
-                                type: "function",
-                                function: {
-                                    name: part.functionCall.name,
-                                    arguments: part.functionCall.args || {},
-                                },
-                            });
-                            yield { delta: { json: { toolCalls } } };
+                            if (part.functionCall.name === OUTPUT_JSON_FUNCTION_NAME) {
+                                json = part.functionCall.args;
+                            }
+                            else {
+                                toolCalls.push({
+                                    id: part.functionCall.id || v7(),
+                                    type: "function",
+                                    function: {
+                                        name: part.functionCall.name,
+                                        arguments: part.functionCall.args || {},
+                                    },
+                                });
+                                yield { delta: { json: { toolCalls } } };
+                            }
                         }
                     }
                 }
             }
             if (chunk.usageMetadata) {
-                usage.inputTokens += chunk.usageMetadata.promptTokenCount || 0;
-                usage.outputTokens += chunk.usageMetadata.candidatesTokenCount || 0;
+                if (chunk.usageMetadata.promptTokenCount)
+                    usage.inputTokens = chunk.usageMetadata.promptTokenCount;
+                if (chunk.usageMetadata.candidatesTokenCount)
+                    usage.outputTokens = chunk.usageMetadata.candidatesTokenCount;
             }
         }
         if (input.responseFormat?.type === "json_schema") {
-            yield { delta: { json: { json: safeParseJSON(text) } } };
+            if (json) {
+                yield { delta: { json: { json } } };
+            }
+            else if (text) {
+                yield { delta: { json: { json: safeParseJSON(text) } } };
+            }
+            else {
+                // NOTE: Trigger retry of chat model
+                throw new StructuredOutputError("No JSON response from the model");
+            }
+        }
+        else if (!toolCalls.length) {
+            if (!text) {
+                logger.error("No text response from the model", parameters);
+                // NOTE: Trigger retry of chat model
+                throw new StructuredOutputError("No text response from the model");
+            }
         }
-        yield { delta: { json: { usage, files } } };
+        yield { delta: { json: { usage, files: files.length ? files : undefined } } };
     }
     async buildConfig(input) {
         const config = {};
+        const { tools, toolConfig } = await this.buildTools(input);
+        config.tools = tools;
+        config.toolConfig = toolConfig;
         if (input.responseFormat?.type === "json_schema") {
-            config.responseJsonSchema = input.responseFormat.jsonSchema.schema;
-            config.responseMimeType = "application/json";
+            if (config.tools?.length) {
+                config.tools.push({
+                    functionDeclarations: [
+                        {
+                            name: OUTPUT_JSON_FUNCTION_NAME,
+                            description: "Output the final response in JSON format",
+                            parametersJsonSchema: input.responseFormat.jsonSchema.schema,
+                        },
+                    ],
+                });
+                config.toolConfig = {
+                    ...config.toolConfig,
+                    functionCallingConfig: { mode: FunctionCallingConfigMode.ANY },
+                };
+            }
+            else {
+                config.responseJsonSchema = input.responseFormat.jsonSchema.schema;
+                config.responseMimeType = "application/json";
+            }
         }
         return config;
     }
@@ -175,7 +225,7 @@ export class GeminiChatModel extends OpenAIChatModel {
                 return;
             }
             const content = {
-                role: msg.role === "agent" ? "model" : "user",
+                role: msg.role === "agent" ? "model" : msg.role === "user" ? "user" : undefined,
             };
             if (msg.toolCalls) {
                 content.parts = msg.toolCalls.map((call) => ({
@@ -192,12 +242,31 @@ export class GeminiChatModel extends OpenAIChatModel {
                     .find((c) => c?.id === msg.toolCallId);
                 if (!call)
                     throw new Error(`Tool call not found: ${msg.toolCallId}`);
+                const output = JSON.parse(msg.content);
+                const isError = "error" in output && Boolean(input.error);
+                const response = {
+                    tool: call.function.name,
+                };
+                // NOTE: base on the documentation of gemini api, the content should include `output` field for successful result or `error` field for failed result,
+                // and base on the actual test, add a tool field presenting the tool name can improve the LLM understanding that which tool is called.
+                if (isError) {
+                    Object.assign(response, { status: "error" }, output);
+                }
+                else {
+                    Object.assign(response, { status: "success" });
+                    if ("output" in output) {
+                        Object.assign(response, output);
+                    }
+                    else {
+                        Object.assign(response, { output });
+                    }
+                }
                 content.parts = [
                     {
                         functionResponse: {
                             id: msg.toolCallId,
                             name: call.function.name,
-                            response: JSON.parse(msg.content),
+                            response,
                         },
                     },
                 ];
@@ -221,23 +290,16 @@ export class GeminiChatModel extends OpenAIChatModel {
             }
             return content;
         }))).filter(isNonNullable);
+        if (!result.contents.length && systemParts.length) {
+            const system = systemParts.pop();
+            if (system) {
+                result.contents.push({ role: "user", parts: [system] });
+            }
+        }
         if (systemParts.length) {
             result.config ??= {};
             result.config.systemInstruction = systemParts;
         }
         return result;
     }
-    async getRunMessages(input) {
-        const messages = await super.getRunMessages(input);
-        if (!messages.some((i) => i.role === "user")) {
-            for (const msg of messages) {
-                if (msg.role === "system") {
-                    // Ensure the last message is from the user
-                    msg.role = "user";
-                    break;
-                }
-            }
-        }
-        return messages;
-    }
 }

package/lib/esm/gemini-image-model.js

@@ -129,6 +129,7 @@ export class GeminiImageModel extends ImageModel {
             "tools",
             "topK",
             "topP",
+            "imageConfig",
         ];
         const images = await Promise.all(flat(input.image).map(async (image) => {
             const { data, mimeType } = await this.transformFileType("file", image, options);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aigne/gemini",
-  "version": "0.14.2-beta",
+  "version": "0.14.2-beta.10",
   "description": "AIGNE Gemini SDK for integrating with Google's Gemini AI models",
   "publishConfig": {
     "access": "public"
@@ -36,9 +36,9 @@
   },
   "dependencies": {
     "@aigne/uuid": "^13.0.1",
-    "@google/genai": "^1.20.0",
+    "@google/genai": "^1.24.0",
     "zod": "^3.25.67",
-    "@aigne/openai": "^0.16.2-beta",
+    "@aigne/core": "^1.63.0-beta.10",
     "@aigne/platform-helpers": "^0.6.3"
   },
   "devDependencies": {
@@ -47,8 +47,7 @@
     "npm-run-all": "^4.1.5",
     "rimraf": "^6.0.1",
     "typescript": "^5.9.2",
-    "@aigne/core": "^1.63.0-beta",
-    "@aigne/test-utils": "^0.5.55-beta"
+    "@aigne/test-utils": "^0.5.55-beta.10"
   },
   "scripts": {
     "lint": "tsc --noEmit",