@mate-academy/llm-gateway 0.0.1-canary.1

package/src/providers/GoogleGenerativeAI/services/GoogleGenerativeAIAssistance.service.ts

@@ -0,0 +1,307 @@
+import { Logger } from '@mate-academy/core';
+import { v4 as uuidv4 } from 'uuid';
+import {
+  Chat,
+  Content,
+  CachedContent,
+  GoogleGenAI,
+} from '@google/genai';
+import {
+  LLMAssistanceOptions,
+  LLMAssistanceResult,
+  LLMCreateAssistantResult,
+  LLMCreateChatOptions,
+  LLMCreateChatResult,
+  LLMCreateFileStorageOptions,
+  LLMCreateFileStorageResult,
+  LLMInstanceOptions,
+  LLMProviders,
+  LLMUploadedFile,
+  LLMUploadFile,
+  LLMUploadFileResult,
+} from '../../../LLMService.typedefs';
+import { LLMAssistanceService } from '../../../services';
+import { GoogleGenerativeAIRoles } from '../GoogleGenerativeAI.typedefs';
+import { GoogleGenerativeAIEntity } from '../GoogleGenerativeAI.entity';
+import { MIN_CACHE_CONTENT_LENGTH } from '../GoogleGenerativeAI.constants';
+
+export class GoogleGenerativeAIAssistanceService
+  extends LLMAssistanceService<LLMProviders.GoogleGenerativeAI> {
+  #instance: GoogleGenAI | null = null;
+
+  private cachedContent: Map<string, CachedContent> = new Map();
+
+  private chatSessions: Map<string, Chat> = new Map();
+
+  private uploadedFiles: Map<string, LLMUploadedFile> = new Map();
+
+  constructor(
+    logger: Logger,
+    options: LLMInstanceOptions[LLMProviders.GoogleGenerativeAI],
+  ) {
+    super(LLMProviders.GoogleGenerativeAI, logger, options);
+  }
+
+  protected get instance(): GoogleGenAI {
+    if (!this.#instance) {
+      this.#instance = new GoogleGenAI({
+        apiKey: this.options.apiKey,
+      });
+    }
+
+    return this.#instance;
+  }
+
+  async uploadFile(file: LLMUploadFile): Promise<LLMUploadFileResult> {
+    try {
+      const {
+        path,
+        mimeType,
+        name,
+      } = file;
+
+      const existingFile = this.uploadedFiles.get(path);
+
+      if (existingFile) {
+        return existingFile;
+      }
+
+      const uploadResult = await this.instance.files.upload({
+        file: path,
+        config: {
+          mimeType,
+          displayName: name,
+        },
+      });
+
+      if (!uploadResult.name) {
+        throw new Error('Upload result did not return a name');
+      }
+
+      if (!uploadResult.uri) {
+        throw new Error('Upload result did not return a uri');
+      }
+
+      if (uploadResult.error) {
+        throw uploadResult.error;
+      }
+
+      const uploadedFile = {
+        ...file,
+        fileId: uploadResult.name,
+        path: uploadResult.uri,
+      };
+
+      this.uploadedFiles.set(path, uploadedFile);
+
+      return uploadedFile;
+    } catch (error: any) {
+      this.logger
+        .child('uploadFile')
+        .error('Error uploading file', {
+          error,
+          file,
+        });
+
+      return { error };
+    }
+  }
+
+  async deleteFile(
+    fileId: string,
+  ): Promise<void> {
+    this.uploadedFiles.delete(fileId);
+
+    await this.instance.files.delete({
+      name: fileId,
+    });
+  }
+
+  async createFileStorage(
+    options: LLMCreateFileStorageOptions<LLMProviders.GoogleGenerativeAI>,
+  ): Promise<LLMCreateFileStorageResult> {
+    const {
+      uploadedFiles,
+      model,
+      instructions,
+    } = options;
+
+    try {
+      const parts = GoogleGenerativeAIEntity.getFileDataParts(uploadedFiles);
+
+      const countTokensFn = GoogleGenerativeAIEntity
+        .getCountTokensFn(this.instance);
+
+      const tokenCount = await countTokensFn(parts, model);
+
+      if (tokenCount < MIN_CACHE_CONTENT_LENGTH) {
+        return {
+          storageId: '',
+        };
+      }
+
+      const cacheResult = await this.instance.caches.create({
+        model: model.name,
+        config: {
+          contents: [
+            {
+              role: GoogleGenerativeAIRoles.User,
+              parts,
+            },
+          ],
+          systemInstruction: instructions,
+          ttl: '1h',
+        },
+      });
+
+      if (!cacheResult.name) {
+        throw new Error('Cache result did not return a name');
+      }
+
+      this.cachedContent.set(cacheResult.name, cacheResult);
+
+      return {
+        storageId: cacheResult.name,
+      };
+    } catch (error: any) {
+      this.logger
+        .child('createFileStorage')
+        .error('Error creating file storage', {
+          error,
+          uploadedFiles,
+          model,
+        });
+
+      return { error };
+    }
+  }
+
+  async deleteFileStorage(
+    fileStorageId: string,
+  ): Promise<void> {
+    if (fileStorageId === '') {
+      return;
+    }
+
+    this.cachedContent.delete(fileStorageId);
+
+    await this.instance.caches.delete({
+      name: fileStorageId,
+    });
+  }
+
+  createChat(
+    options: LLMCreateChatOptions<LLMProviders.GoogleGenerativeAI>,
+  ): Promise<LLMCreateChatResult> {
+    try {
+      const {
+        storageId,
+        model,
+        instructions,
+        history: providedHistory,
+        files,
+      } = options;
+
+      const cachedContent = storageId
+        ? this.cachedContent.get(storageId)
+        : undefined;
+
+      const history: Content[] = providedHistory
+        ? [
+          {
+            role: GoogleGenerativeAIRoles.User,
+            parts: GoogleGenerativeAIEntity.getTextParts(providedHistory),
+          },
+        ]
+        : [];
+
+      if (!cachedContent && files) {
+        const fileParts = GoogleGenerativeAIEntity.getFileDataParts(files);
+
+        history.push({
+          role: GoogleGenerativeAIRoles.User,
+          parts: fileParts,
+        });
+      }
+
+      const chatSession = this.instance.chats.create({
+        model: model.name,
+        history,
+        config: {
+          systemInstruction: instructions,
+          temperature: model.config.temperature,
+          ...(cachedContent && { cachedContent: cachedContent.name }),
+        },
+      });
+
+      const chatId = uuidv4();
+
+      this.chatSessions.set(chatId, chatSession);
+
+      return Promise.resolve({ chatId });
+    } catch (error: any) {
+      this.logger
+        .child('createChat')
+        .error('Error creating chat', {
+          error,
+          options,
+        });
+
+      return Promise.resolve({ error });
+    }
+  }
+
+  deleteChat(
+    chatId: string,
+  ): Promise<void> {
+    this.chatSessions.delete(chatId);
+
+    return Promise.resolve();
+  }
+
+  createAssistant(): Promise<LLMCreateAssistantResult> {
+    // Google Generative AI does not have an assistant concept at the moment
+    return Promise.resolve({
+      assistantId: '',
+    });
+  }
+
+  deleteAssistant(): Promise<void> {
+    // Google Generative AI does not have an assistant concept at the moment
+    return Promise.resolve();
+  }
+
+  async assistInChat(
+    options: LLMAssistanceOptions<LLMProviders.GoogleGenerativeAI>,
+  ): Promise<LLMAssistanceResult> {
+    try {
+      const {
+        message,
+        chatId,
+      } = options;
+
+      const chatSession = this.chatSessions.get(chatId);
+
+      if (!chatSession) {
+        throw new Error('Chat session not found');
+      }
+
+      const chatResponse = await chatSession.sendMessage({
+        message: message.content,
+      });
+
+      return {
+        text: chatResponse.text ?? '',
+      };
+    } catch (error: any) {
+      this.logger
+        .child('assistInChat')
+        .error('Error assisting in chat', {
+          error,
+          options,
+        });
+
+      return { error };
+    }
+  }
+}

package/src/providers/GoogleGenerativeAI/services/GoogleGenerativeAICompletion.service.ts

@@ -0,0 +1,79 @@
+import { Logger } from '@mate-academy/core';
+import { GoogleGenAI } from '@google/genai';
+import {
+  LLMCompletionResult,
+  LLMInstanceOptions,
+  LLMProviders,
+  LLMSendMessageOptions,
+} from '../../../LLMService.typedefs';
+import {
+  LLMCompletionService,
+} from '../../../services';
+import { LLMServiceEntity } from '../../../LLMService.entity';
+import { GoogleGenerativeAIEntity } from '../GoogleGenerativeAI.entity';
+
+export class GoogleGenerativeAICompletionService
+  extends LLMCompletionService<LLMProviders.GoogleGenerativeAI> {
+  #instance: GoogleGenAI | null = null;
+
+  constructor(
+    logger: Logger,
+    options: LLMInstanceOptions[LLMProviders.GoogleGenerativeAI],
+  ) {
+    super(LLMProviders.GoogleGenerativeAI, logger, options);
+  }
+
+  protected get instance(): GoogleGenAI {
+    if (!this.#instance) {
+      this.#instance = new GoogleGenAI({
+        apiKey: this.options.apiKey,
+      });
+    }
+
+    return this.#instance;
+  }
+
+  async sendMessage(
+    options: LLMSendMessageOptions<typeof this.provider>,
+  ): Promise<LLMCompletionResult> {
+    try {
+      const {
+        model,
+        message,
+        history = [],
+        instructions,
+      } = options;
+
+      const countTokensFn = GoogleGenerativeAIEntity
+        .getCountTokensFn(this.instance);
+
+      const combinedMessages = await LLMServiceEntity.combineMessagesWithLimit({
+        history,
+        message: message.content,
+        model,
+        countTokensFn,
+      });
+
+      const result = await this.instance.models.generateContent(
+        GoogleGenerativeAIEntity.getContentParameters(
+          model,
+          combinedMessages,
+          instructions,
+        ),
+      );
+
+      return {
+        text: result.text ?? '',
+      };
+    } catch (error: any) {
+      this.logger
+        .child('sendMessage')
+        .error('Error sending message', {
+          error,
+          options,
+        });
+
+      return { error };
+    }
+  }
+}

package/src/providers/GoogleGenerativeAI/services/index.ts

@@ -0,0 +1,2 @@
+export * from './GoogleGenerativeAICompletion.service';
+export * from './GoogleGenerativeAIAssistance.service';

package/src/providers/OpenAI/OpenAI.constants.ts

@@ -0,0 +1,71 @@
+import {
+  LLMProviderAvailableModels,
+  LLMProviderModelsByPurpose,
+  LLMProviders,
+  LLMPurposes,
+  LLMServiceBuilder,
+} from '../../LLMService.typedefs';
+import { OpenAIModelNames } from './OpenAI.typedefs';
+import {
+  OpenAIAssistanceService,
+  OpenAICompletionService,
+} from './services';
+import { pick } from '../../functional.utils';
+
+const OPEN_AI_AVAILABLE_MODELS: LLMProviderAvailableModels<
+  LLMProviders.OpenAI
+> = {
+  [OpenAIModelNames.GPT_4_OMNI]: {
+    name: OpenAIModelNames.GPT_4_OMNI,
+    limits: {
+      maxInputTokens: 128_000,
+      maxOutputTokens: 16_384,
+    },
+    config: {
+      temperature: 0.2,
+    },
+  },
+  [OpenAIModelNames.GPT_4_1]: {
+    name: OpenAIModelNames.GPT_4_1,
+    limits: {
+      maxInputTokens: 1_047_576,
+      maxOutputTokens: 32_768,
+    },
+    config: {
+      temperature: 0.2,
+    },
+  },
+};
+
+export const OPEN_AI_MODELS: LLMProviderModelsByPurpose<
+  LLMPurposes,
+  LLMProviders.OpenAI
+> = {
+  [LLMPurposes.Completion]: pick(
+    OPEN_AI_AVAILABLE_MODELS,
+    [
+      OpenAIModelNames.GPT_4_OMNI,
+      OpenAIModelNames.GPT_4_1,
+    ],
+  ),
+  [LLMPurposes.Assistance]: pick(
+    OPEN_AI_AVAILABLE_MODELS,
+    [
+      OpenAIModelNames.GPT_4_OMNI,
+      OpenAIModelNames.GPT_4_1,
+    ],
+  ),
+};
+
+export const OPEN_AI_SERVICE_BUILDERS: {
+  [purpose in LLMPurposes]: (
+    LLMServiceBuilder<LLMProviders.OpenAI, purpose> | null
+  )
+} = {
+  [LLMPurposes.Completion]: (logger, options) => (
+    new OpenAICompletionService(logger, options)
+  ),
+  [LLMPurposes.Assistance]: (logger, options) => (
+    new OpenAIAssistanceService(logger, options)
+  ),
+};

package/src/providers/OpenAI/OpenAI.entity.ts

@@ -0,0 +1,17 @@
+import { getEncoding } from 'js-tiktoken';
+
+export class OpenAIEntity {
+  static getCountTokensFn() {
+    return (
+      messages: string[],
+    ): Promise<number> => {
+      const encoder = getEncoding('o200k_base');
+      const tokens = messages.reduce(
+        (acc, message) => acc + encoder.encode(message).length,
+        0,
+      );
+
+      return Promise.resolve(tokens);
+    };
+  }
+}

package/src/providers/OpenAI/OpenAI.typedefs.ts

@@ -0,0 +1,9 @@
+export enum OpenAIModelNames {
+  GPT_4_OMNI = 'gpt-4o',
+  GPT_4_1 = 'gpt-4.1',
+}
+
+export enum OpenAIRoles {
+  User = 'user',
+  Assistant = 'assistant',
+}

package/src/providers/OpenAI/OpenAIService.factory.ts

@@ -0,0 +1,29 @@
+import { Logger } from '@mate-academy/core';
+import {
+  LLMInstanceOptions,
+  LLMProviders,
+  LLMPurposes,
+  LLMServiceByPurpose,
+} from '../../LLMService.typedefs';
+import { LLMServicePurposeFactory } from '../../services';
+import { OPEN_AI_SERVICE_BUILDERS } from './OpenAI.constants';
+
+export class OpenAIServiceFactory extends LLMServicePurposeFactory<
+  LLMProviders.OpenAI
+> {
+  createService<
+    Purpose extends LLMPurposes
+  >(
+    purpose: Purpose,
+    logger: Logger,
+    options: LLMInstanceOptions[LLMProviders.OpenAI],
+  ): LLMServiceByPurpose<LLMProviders.OpenAI>[Purpose] {
+    const serviceBuilder = OPEN_AI_SERVICE_BUILDERS[purpose];
+
+    if (!serviceBuilder) {
+      throw new Error(`Purpose [${purpose}] is not supported for [${LLMProviders.OpenAI}] service`);
+    }
+
+    return serviceBuilder(logger, options);
+  }
+}