@robota-sdk/agent-provider 3.0.0-beta.64

package/src/gemini/provider.spec.ts

@@ -0,0 +1,216 @@
+import { describe, expect, it, vi, beforeEach } from 'vitest';
+import { GeminiProvider } from './provider';
+import type { TUniversalMessage } from '@robota-sdk/agent-core';
+
+interface IGenerateContentInput {
+  model: string;
+  contents: Array<{
+    role: 'user' | 'model';
+    parts: Array<{
+      text?: string;
+      inlineData?: {
+        mimeType: string;
+        data: string;
+      };
+      functionCall?: {
+        id?: string;
+        name: string;
+        args: Record<string, string | number | boolean | object>;
+      };
+    }>;
+  }>;
+  config?: {
+    temperature?: number;
+    maxOutputTokens?: number;
+    responseModalities?: Array<'TEXT' | 'IMAGE'>;
+  };
+}
+
+const generateContentMock = vi.fn<
+  [IGenerateContentInput],
+  Promise<{
+    candidates: Array<{
+      content: {
+        parts: Array<{
+          text?: string;
+          inlineData?: {
+            mimeType: string;
+            data: string;
+          };
+        }>;
+      };
+    }>;
+  }>
+>();
+
+vi.mock('@google/genai', () => {
+  class GoogleGenAI {
+    public readonly models = {
+      generateContent: generateContentMock,
+      async *generateContentStream(): AsyncIterable<{ text: string }> {
+        yield { text: '' };
+      },
+    };
+
+    public constructor(_options: { apiKey: string }) {}
+  }
+  return {
+    GoogleGenAI,
+    Type: {
+      STRING: 'STRING',
+      NUMBER: 'NUMBER',
+      INTEGER: 'INTEGER',
+      BOOLEAN: 'BOOLEAN',
+      ARRAY: 'ARRAY',
+      OBJECT: 'OBJECT',
+    },
+  };
+});
+
+describe('GeminiProvider image support', () => {
+  beforeEach(() => {
+    generateContentMock.mockReset();
+  });
+
+  it('maps inline image output from Gemini response to assistant parts', async () => {
+    generateContentMock.mockResolvedValue({
+      candidates: [
+        {
+          content: {
+            parts: [
+              { text: 'created' },
+              {
+                inlineData: {
+                  mimeType: 'image/png',
+                  data: 'ZmFrZS1pbWFnZS1kYXRh',
+                },
+              },
+            ],
+          },
+        },
+      ],
+    });
+
+    const provider = new GeminiProvider({ apiKey: 'test-key' });
+    const response = await provider.chat(
+      [
+        {
+          id: 'msg-1',
+          state: 'complete' as const,
+          role: 'user',
+          content: 'create an image',
+          timestamp: new Date(),
+        },
+      ],
+      {
+        model: 'gemini-2.5-flash-image',
+        google: { responseModalities: ['TEXT', 'IMAGE'] },
+      },
+    );
+
+    expect(response.role).toBe('assistant');
+    expect(response.parts).toBeDefined();
+    expect(response.parts?.some((part) => part.type === 'image_inline')).toBe(true);
+  });
+
+  it('maps inline image input parts into Gemini inlineData request parts', async () => {
+    generateContentMock.mockResolvedValue({
+      candidates: [
+        {
+          content: {
+            parts: [
+              { text: 'ok' },
+              {
+                inlineData: {
+                  mimeType: 'image/png',
+                  data: 'ZmFrZS1pbWFnZS1kYXRh',
+                },
+              },
+            ],
+          },
+        },
+      ],
+    });
+
+    const provider = new GeminiProvider({ apiKey: 'test-key' });
+    const messages: TUniversalMessage[] = [
+      {
+        id: 'msg-1',
+        state: 'complete' as const,
+        role: 'user',
+        content: '',
+        parts: [
+          {
+            type: 'image_inline',
+            mimeType: 'image/png',
+            data: 'ZmFrZS1pbWFnZS1pbnB1dA==',
+          },
+          {
+            type: 'text',
+            text: 'edit this image',
+          },
+        ],
+        timestamp: new Date(),
+      },
+    ];
+
+    await provider.chat(messages, {
+      model: 'gemini-2.5-flash-image',
+    });
+
+    const calledPayload = generateContentMock.mock.calls[0]?.[0];
+    expect(calledPayload.contents[0]?.parts[0]?.inlineData?.mimeType).toBe('image/png');
+    expect(calledPayload.contents[0]?.parts[1]?.text).toBe('edit this image');
+    expect(calledPayload.config?.responseModalities).toEqual(['TEXT', 'IMAGE']);
+  });
+
+  it('throws when image modality is requested with non-image model', async () => {
+    const provider = new GeminiProvider({ apiKey: 'test-key' });
+
+    await expect(
+      provider.chat(
+        [
+          {
+            id: 'msg-1',
+            state: 'complete' as const,
+            role: 'user',
+            content: 'generate image',
+            timestamp: new Date(),
+          },
+        ],
+        {
+          model: 'gemini-1.5-pro',
+          google: { responseModalities: ['IMAGE'] },
+        },
+      ),
+    ).rejects.toThrow('Google chat failed:');
+  });
+
+  it('throws when image uri message part is used directly', async () => {
+    const provider = new GeminiProvider({ apiKey: 'test-key' });
+
+    await expect(
+      provider.chat(
+        [
+          {
+            id: 'msg-1',
+            state: 'complete' as const,
+            role: 'user',
+            content: '',
+            parts: [
+              {
+                type: 'image_uri',
+                uri: 'asset://example',
+                mimeType: 'image/png',
+              },
+            ],
+            timestamp: new Date(),
+          },
+        ],
+        {
+          model: 'gemini-2.5-flash-image',
+        },
+      ),
+    ).rejects.toThrow('Google chat failed:');
+  });
+});

package/src/gemini/provider.ts

@@ -0,0 +1,279 @@
+import { randomUUID } from 'node:crypto';
+import { GoogleGenAI } from '@google/genai';
+import type { IGeminiProviderOptions } from './types';
+import { AbstractAIProvider } from '@robota-sdk/agent-core';
+import type {
+  TUniversalMessage,
+  IChatOptions,
+  TTextDeltaCallback,
+  TUniversalMessagePart,
+  IImageGenerationProvider,
+  IImageGenerationRequest,
+  IImageEditRequest,
+  IImageComposeRequest,
+  IImageGenerationResult,
+  TProviderMediaResult,
+} from '@robota-sdk/agent-core';
+import { mapImageInputSourceToPart } from './image-operations';
+import { executeDirect, executeDirectStream, runImageRequest } from './execution-helpers';
+
+/**
+ * Gemini provider implementation for Robota
+ *
+ * IMPORTANT PROVIDER-SPECIFIC RULES:
+ * 1. This provider MUST extend BaseAIProvider from @robota-sdk/agent-core
+ * 2. Content handling for Google Gemini API:
+ *    - Function calls can have content (text) along with function calls
+ *    - Content can be empty string or actual text, NOT null
+ * 3. Use override keyword for all methods inherited from BaseAIProvider
+ * 4. Provider-specific API behavior should be documented here
+ *
+ * @public
+ */
+export class GeminiProvider extends AbstractAIProvider implements IImageGenerationProvider {
+  override readonly name: string = 'gemini';
+  override readonly version = '1.0.0';
+  public onTextDelta?: TTextDeltaCallback;
+
+  private readonly client?: GoogleGenAI;
+  private readonly options: IGeminiProviderOptions;
+
+  constructor(options: IGeminiProviderOptions) {
+    super();
+    this.options = options;
+
+    if (options.executor) {
+      this.executor = options.executor;
+    }
+
+    if (!this.executor) {
+      this.client = new GoogleGenAI({ apiKey: options.apiKey });
+    }
+  }
+
+  /** Generate response using TUniversalMessage */
+  override async chat(
+    messages: TUniversalMessage[],
+    options?: IChatOptions,
+  ): Promise<TUniversalMessage> {
+    this.validateMessages(messages);
+
+    if (this.executor) {
+      try {
+        return await this.executeViaExecutorOrDirect(messages, options);
+      } catch (error) {
+        this.logger.error(
+          'Gemini Provider executor chat error:',
+          error instanceof Error ? error.message : String(error),
+        );
+        throw error;
+      }
+    }
+
+    if (!this.client) {
+      throw new Error('Google client not available. Either provide apiKey or use an executor.');
+    }
+
+    try {
+      return await executeDirect(
+        this.client,
+        this.options,
+        messages,
+        this.withProviderCallbacks(options),
+        this.name,
+      );
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Google API request failed';
+      throw new Error(`Google chat failed: ${errorMessage}`);
+    }
+  }
+
+  /** Generate streaming response using TUniversalMessage */
+  override async *chatStream(
+    messages: TUniversalMessage[],
+    options?: IChatOptions,
+  ): AsyncIterable<TUniversalMessage> {
+    this.validateMessages(messages);
+    if (this.executor) {
+      try {
+        yield* this.executeStreamViaExecutorOrDirect(messages, options);
+        return;
+      } catch (error) {
+        this.logger.error(
+          'Gemini Provider executor stream error:',
+          error instanceof Error ? error.message : String(error),
+        );
+        throw error;
+      }
+    }
+
+    if (!this.client) {
+      throw new Error('Google client not available. Either provide apiKey or use an executor.');
+    }
+
+    try {
+      yield* executeDirectStream(
+        this.client,
+        this.options,
+        messages,
+        this.withProviderCallbacks(options),
+        this.name,
+      );
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : 'Google API request failed';
+      throw new Error(`Google stream failed: ${errorMessage}`);
+    }
+  }
+
+  /** Generate an image from a text prompt using the Gemini API. */
+  public async generateImage(
+    request: IImageGenerationRequest,
+  ): Promise<TProviderMediaResult<IImageGenerationResult>> {
+    if (request.prompt.trim().length === 0) {
+      return {
+        ok: false,
+        error: {
+          code: 'PROVIDER_INVALID_REQUEST',
+          message: 'Image generation requires a non-empty prompt.',
+        },
+      };
+    }
+    if (request.model.trim().length === 0) {
+      return {
+        ok: false,
+        error: {
+          code: 'PROVIDER_INVALID_REQUEST',
+          message: 'Image generation requires a non-empty model.',
+        },
+      };
+    }
+
+    const message: TUniversalMessage = {
+      id: randomUUID(),
+      role: 'user',
+      content: request.prompt,
+      state: 'complete' as const,
+      parts: [{ type: 'text', text: request.prompt }],
+      timestamp: new Date(),
+    };
+    return runImageRequest(this.chat.bind(this), [message], request.model);
+  }
+
+  /** Edit an existing image based on a text prompt using the Gemini API. */
+  public async editImage(
+    request: IImageEditRequest,
+  ): Promise<TProviderMediaResult<IImageGenerationResult>> {
+    if (request.prompt.trim().length === 0) {
+      return {
+        ok: false,
+        error: {
+          code: 'PROVIDER_INVALID_REQUEST',
+          message: 'Image edit requires a non-empty prompt.',
+        },
+      };
+    }
+    if (request.model.trim().length === 0) {
+      return {
+        ok: false,
+        error: {
+          code: 'PROVIDER_INVALID_REQUEST',
+          message: 'Image edit requires a non-empty model.',
+        },
+      };
+    }
+
+    const inputPartResult = mapImageInputSourceToPart(request.image);
+    if (!inputPartResult.ok) {
+      return inputPartResult;
+    }
+
+    const message: TUniversalMessage = {
+      id: randomUUID(),
+      role: 'user',
+      content: request.prompt,
+      state: 'complete' as const,
+      parts: [inputPartResult.value, { type: 'text', text: request.prompt }],
+      timestamp: new Date(),
+    };
+    return runImageRequest(this.chat.bind(this), [message], request.model);
+  }
+
+  /** Compose multiple images together based on a text prompt using the Gemini API. */
+  public async composeImage(
+    request: IImageComposeRequest,
+  ): Promise<TProviderMediaResult<IImageGenerationResult>> {
+    if (request.prompt.trim().length === 0) {
+      return {
+        ok: false,
+        error: {
+          code: 'PROVIDER_INVALID_REQUEST',
+          message: 'Image compose requires a non-empty prompt.',
+        },
+      };
+    }
+    if (request.model.trim().length === 0) {
+      return {
+        ok: false,
+        error: {
+          code: 'PROVIDER_INVALID_REQUEST',
+          message: 'Image compose requires a non-empty model.',
+        },
+      };
+    }
+    if (request.images.length < 2) {
+      return {
+        ok: false,
+        error: {
+          code: 'PROVIDER_INVALID_REQUEST',
+          message: 'Image compose requires at least two input images.',
+        },
+      };
+    }
+
+    const messageParts: TUniversalMessagePart[] = [];
+    for (const imageSource of request.images) {
+      const mappedPartResult = mapImageInputSourceToPart(imageSource);
+      if (!mappedPartResult.ok) {
+        return mappedPartResult;
+      }
+      messageParts.push(mappedPartResult.value);
+    }
+    messageParts.push({ type: 'text', text: request.prompt });
+
+    const message: TUniversalMessage = {
+      id: randomUUID(),
+      role: 'user',
+      content: request.prompt,
+      state: 'complete' as const,
+      parts: messageParts,
+      timestamp: new Date(),
+    };
+    return runImageRequest(this.chat.bind(this), [message], request.model);
+  }
+
+  override supportsTools(): boolean {
+    return true;
+  }
+
+  override validateConfig(): boolean {
+    if (this.executor) {
+      return this.executor.validateConfig();
+    }
+    return !!this.client && !!this.options && !!this.options.apiKey;
+  }
+
+  override async dispose(): Promise<void> {
+    // Google client does not need explicit cleanup
+  }
+
+  private withProviderCallbacks(options?: IChatOptions): IChatOptions | undefined {
+    const onTextDelta = options?.onTextDelta ?? this.onTextDelta;
+    if (!onTextDelta) {
+      return options;
+    }
+    return {
+      ...options,
+      onTextDelta,
+    };
+  }
+}

package/src/gemini/request-converter.ts

@@ -0,0 +1,226 @@
+import type { Content, Part } from '@google/genai';
+import type {
+  IAssistantMessage,
+  ISystemMessage,
+  IToolMessage,
+  IUserMessage,
+  TUniversalMessage,
+} from '@robota-sdk/agent-core';
+
+type TGoogleJsonValue = string | number | boolean | null | TGoogleJsonValue[] | IGoogleJsonObject;
+
+interface IGoogleJsonObject {
+  readonly [key: string]: TGoogleJsonValue;
+}
+
+export interface IGeminiMessageConversionResult {
+  contents: Content[];
+  systemInstruction?: string;
+}
+
+/**
+ * Maps universal message parts to Gemini-compatible parts.
+ * Supports text and inline image parts; throws on unsupported part types.
+ */
+export function mapMessagePartsToGeminiParts(
+  message: IUserMessage | IAssistantMessage | ISystemMessage | IToolMessage,
+): Part[] {
+  const parts: Part[] = [];
+  const messageParts = message.parts ?? [];
+  for (const part of messageParts) {
+    if (part.type === 'text') {
+      parts.push({ text: part.text });
+      continue;
+    }
+    if (part.type === 'image_inline') {
+      parts.push({
+        inlineData: {
+          mimeType: part.mimeType,
+          data: part.data,
+        },
+      });
+      continue;
+    }
+    throw new Error(`Google provider does not support image URI parts directly: ${part.uri}`);
+  }
+  if (parts.length === 0 && typeof message.content === 'string' && message.content.length > 0) {
+    parts.push({ text: message.content });
+  }
+  return parts;
+}
+
+/**
+ * Converts an array of universal messages to the Gemini Content format.
+ *
+ * IMPORTANT: Google Gemini allows content with function calls.
+ * Content can be empty string or text, but NOT null.
+ */
+export function convertToGeminiFormat(messages: TUniversalMessage[]): Content[] {
+  return convertToGeminiRequestFormat(messages).contents;
+}
+
+/**
+ * Converts universal messages into Gemini request content plus request config
+ * fields. Gemini system instructions are request-level config, not user turns.
+ */
+export function convertToGeminiRequestFormat(
+  messages: TUniversalMessage[],
+): IGeminiMessageConversionResult {
+  const contents: Content[] = [];
+  const systemInstructionParts: string[] = [];
+
+  for (const msg of messages) {
+    if (msg.role === 'user') {
+      contents.push({
+        role: 'user',
+        parts: mapMessagePartsToGeminiParts(msg as IUserMessage),
+      });
+      continue;
+    }
+    if (msg.role === 'assistant') {
+      contents.push(convertAssistantMessage(msg as IAssistantMessage));
+      continue;
+    }
+    if (msg.role === 'tool') {
+      contents.push(convertToolMessage(msg as IToolMessage));
+      continue;
+    }
+
+    const systemInstruction = extractSystemInstructionText(msg as ISystemMessage);
+    if (systemInstruction.length > 0) {
+      systemInstructionParts.push(systemInstruction);
+    }
+  }
+
+  return {
+    contents,
+    ...(systemInstructionParts.length > 0 && {
+      systemInstruction: systemInstructionParts.join('\n'),
+    }),
+  };
+}
+
+/**
+ * Converts all messages to Gemini contents, including system instructions as
+ * user content. This exists only for compatibility with callers that still need
+ * a contents-only value.
+ */
+export function convertToGeminiFormatWithInlineSystem(messages: TUniversalMessage[]): Content[] {
+  return messages.map((msg) => {
+    if (msg.role === 'user') {
+      return {
+        role: 'user',
+        parts: mapMessagePartsToGeminiParts(msg as IUserMessage),
+      };
+    }
+    if (msg.role === 'assistant') {
+      return convertAssistantMessage(msg as IAssistantMessage);
+    }
+    if (msg.role === 'tool') {
+      const toolMessage = msg as IToolMessage;
+      return {
+        role: 'user',
+        parts: mapMessagePartsToGeminiParts(toolMessage),
+      };
+    }
+    const systemMessage = msg as ISystemMessage;
+    const systemParts = mapMessagePartsToGeminiParts(systemMessage);
+    if (systemParts.length === 0) {
+      systemParts.push({ text: `System: ${systemMessage.content || ''}` });
+    }
+    return {
+      role: 'user',
+      parts: systemParts,
+    };
+  });
+}
+
+function convertAssistantMessage(assistantMsg: IAssistantMessage): Content {
+  const parts: Part[] = [];
+  const mappedAssistantParts = mapMessagePartsToGeminiParts(assistantMsg);
+  for (const mappedPart of mappedAssistantParts) {
+    parts.push(mappedPart);
+  }
+  if (parts.length === 0 && assistantMsg.content) {
+    parts.push({ text: assistantMsg.content });
+  }
+  if (assistantMsg.toolCalls && assistantMsg.toolCalls.length > 0) {
+    assistantMsg.toolCalls.forEach((tc) => {
+      parts.push({
+        functionCall: {
+          id: tc.id,
+          name: tc.function.name,
+          args: parseToolCallArguments(tc.function.arguments),
+        },
+      });
+    });
+  }
+  return {
+    role: 'model',
+    parts,
+  };
+}
+
+function convertToolMessage(toolMessage: IToolMessage): Content {
+  const functionResponse = {
+    id: toolMessage.toolCallId,
+    name: requireToolMessageName(toolMessage),
+    response: parseToolResponseContent(toolMessage.content),
+  };
+  return {
+    role: 'user',
+    parts: [{ functionResponse }],
+  };
+}
+
+function extractSystemInstructionText(systemMessage: ISystemMessage): string {
+  const parts = mapMessagePartsToGeminiParts(systemMessage);
+  if (parts.length === 0) {
+    return systemMessage.content;
+  }
+  const textParts: string[] = [];
+  for (const part of parts) {
+    if (typeof part.text === 'string') {
+      textParts.push(part.text);
+      continue;
+    }
+    throw new Error('Google provider system instructions support only text parts.');
+  }
+  return textParts.join('\n');
+}
+
+function requireToolMessageName(toolMessage: IToolMessage): string {
+  const toolName = toolMessage.name?.trim();
+  if (!toolName) {
+    throw new Error('Google provider tool message requires a function name.');
+  }
+  return toolName;
+}
+
+function parseToolCallArguments(serializedArguments: string): IGoogleJsonObject {
+  const parsedArguments = JSON.parse(serializedArguments) as TGoogleJsonValue;
+  if (!isJsonObject(parsedArguments)) {
+    throw new Error('Google provider tool call arguments must be a JSON object.');
+  }
+  return parsedArguments;
+}
+
+function parseToolResponseContent(content: string): IGoogleJsonObject {
+  const trimmedContent = content.trim();
+  if (trimmedContent.length === 0) {
+    return { output: null };
+  }
+  try {
+    const parsedContent = JSON.parse(trimmedContent) as TGoogleJsonValue;
+    if (isJsonObject(parsedContent)) {
+      return parsedContent;
+    }
+    return { output: parsedContent };
+  } catch {
+    return { output: content };
+  }
+}
+
+function isJsonObject(value: TGoogleJsonValue): value is IGoogleJsonObject {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}