llm-proxy 1.0.0

package/jest.config.js

@@ -0,0 +1,10 @@
+// jest.config.js
+module.exports = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  transform: {
+    "^.+\\.tsx?$": "ts-jest", // Transforms TypeScript files using ts-jest
+  },
+  testMatch: ["**/src/test/**/*.test.ts"], // Updated to match src/test location
+  moduleFileExtensions: ["ts", "js"], // Recognizes ts and js files
+};

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "llm-proxy",
+  "version": "1.0.0",
+  "description": "An LLM Proxy that allows the  user to interact with different language models from different providers using unified request and response formats.",
+  "main": "index.js",
+  "scripts": {
+    "test": "jest --config jest.config.js",
+    "dev": "ts-node src/index.ts"
+  },
+  "keywords": [],
+  "author": "Ahmad Jawabreh - jawabreh0",
+  "license": "MIT",
+  "devDependencies": {
+    "@types/dotenv": "^8.2.3",
+    "@types/jest": "^29.5.14",
+    "@types/node": "^22.8.6",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.2.5",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.6.3"
+  },
+  "dependencies": {
+    "@anthropic-ai/bedrock-sdk": "^0.11.1",
+    "aws-sdk": "^2.1691.0",
+    "axios": "^1.7.7",
+    "dotenv": "^16.4.5",
+    "openai": "^4.69.0"
+  }
+}

package/src/clients/AwsBedrockAnthropicClient.ts

@@ -0,0 +1,56 @@
+// import { AwsBedrockAnthropicService } from "../services/AwsBedrockAnthropicService";
+// import { BedrockAnthropicParsedChunk, Messages, SupportedLLMs } from "../types";
+
+// export class AwsBedrockAnthropicChatClient {
+//   private client: AwsBedrockAnthropicService;
+
+//   constructor() {
+//     this.client = new AwsBedrockAnthropicService();
+//   }
+
+//   async sendMessage(
+//     messages: Messages,
+//     model: SupportedLLMs,
+//     maxTokens?: number,
+//     temperature?: number,
+//     systemPrompt?: string,
+//     tools?: any
+//   ) {
+//     const response = await this.client.generateCompletion(
+//       messages,
+//       model,
+//       maxTokens,
+//       temperature,
+//       systemPrompt,
+//       tools
+//     );
+//     console.log("Response:", response);
+//     return response;
+//   }
+
+//   async *sendMessageStream(
+//     messages: Messages,
+//     model: SupportedLLMs,
+//     maxTokens?: number,
+//     temperature?: number,
+//     systemPrompt?: string,
+//     tools?: any
+//   ): AsyncGenerator<string, void, unknown> {
+//     const stream = this.client.generateStreamCompletion(
+//       messages,
+//       model,
+//       maxTokens,
+//       temperature,
+//       systemPrompt,
+//       tools,
+//       true
+//     );
+
+//     for await (const chunk of stream) {
+//       const content = chunk.delta?.text;
+//       if (content) {
+//         yield content;
+//       }
+//     }
+//   }
+// }

package/src/clients/OpenAIClient.ts

@@ -0,0 +1,66 @@
+// import { OpenAIService } from "../services/OpenAIService";
+// import { OpenAIMessage, OpenAIMessagesRoles, OpenAIResponse, OpenAISupportedLLMs } from "../types";
+
+// export class OpenAIClient {
+//   private client: OpenAIService;
+//   private messages: OpenAIMessage[] = [];
+
+//   constructor(apiKey: string, systemPrompt: string) {
+//     this.client = new OpenAIService(apiKey);
+//     this.messages.push({
+//       role: OpenAIMessagesRoles.SYSTEM,
+//       content: systemPrompt,
+//     });
+//   }
+
+//   async sendMessage(
+//     userInput: string,
+//     model: OpenAISupportedLLMs,
+//     maxTokens: number,
+//     temperature: number
+//   ): Promise<string> {
+//     this.messages.push({ role: OpenAIMessagesRoles.USER, content: userInput });
+
+//     const response: OpenAIResponse = await this.client.generateCompletion(
+//       this.messages,
+//       model,
+//       maxTokens,
+//       temperature
+//     );
+
+//     const responseContent: string = response.choices[0].message.content;
+//     this.messages.push({
+//       role: OpenAIMessagesRoles.ASSISTANT,
+//       content: responseContent,
+//     });
+
+//     return responseContent;
+//   }
+
+//   async *sendMessageStream(
+//     userInput: string,
+//     model: OpenAISupportedLLMs,
+//     maxTokens: number,
+//     temperature: number
+//   ): AsyncGenerator<string, void, unknown> {
+//     this.messages.push({ role: OpenAIMessagesRoles.USER, content: userInput });
+
+//     const stream = this.client.generateStreamCompletion(
+//       this.messages,
+//       model,
+//       maxTokens,
+//       temperature
+//     );
+
+//     for await (const chunk of stream) {
+//       if (
+//         chunk.choices &&
+//         chunk.choices[0].delta &&
+//         chunk.choices[0].delta.content
+//       ) {
+//         const content = chunk.choices[0].delta.content;
+//         yield content;
+//       }
+//     }
+//   }
+// }

package/src/index.ts

@@ -0,0 +1,145 @@
+import { ProviderFinder } from "./middleware/ProviderFinder";
+import { InputFormatAdapter } from "./middleware/InputFormatAdapter";
+import { OutputFormatAdapter } from "./middleware/OutputFormatAdapter";
+import { AwsBedrockAnthropicService } from "./services/AwsBedrockAnthropicService";
+import { OpenAIService } from "./services/OpenAIService";
+import {
+  Messages,
+  SupportedLLMs,
+  LLMResponse,
+  Providers,
+  OpenAIMessages,
+  BedrockAnthropicMessages,
+} from "./types";
+
+// Define the credentials interface for flexibility
+interface Credentials {
+  apiKey?: string;
+  awsConfig?: { accessKeyId: string; secretAccessKey: string; region: string };
+}
+
+// Main function to handle both streaming and non-streaming requests
+export async function generateLLMResponse(
+  messages: Messages,
+  model: SupportedLLMs,
+  maxTokens: number,
+  temperature: number,
+  systemPrompt: string,
+  tools: any,
+  stream: boolean = false,
+  credentials: Credentials
+): Promise<LLMResponse | AsyncGenerator<LLMResponse>> {
+  // Step 2: Identify the provider based on the model
+  const provider = ProviderFinder.getProvider(model);
+
+  // Initialize the correct service based on the provider
+  let service: OpenAIService | AwsBedrockAnthropicService;
+  if (provider === Providers.OPENAI) {
+    if (!credentials.apiKey) {
+      throw new Error("OpenAI API key is required for OpenAI models.");
+    }
+    service = new OpenAIService(credentials.apiKey);
+  } else if (provider === Providers.ANTHROPIC_BEDROCK) {
+    const awsConfig = credentials.awsConfig;
+    if (!awsConfig) {
+      throw new Error("AWS credentials are required for Bedrock models.");
+    }
+    service = new AwsBedrockAnthropicService(
+      awsConfig.accessKeyId,
+      awsConfig.secretAccessKey,
+      awsConfig.region
+    );
+  } else {
+    throw new Error("Unsupported provider");
+  }
+
+  // Step 3: If the provider is not OpenAI, adapt the input to provider format
+  const adaptedMessages =
+    provider !== Providers.OPENAI
+      ? InputFormatAdapter.adaptMessages(messages, provider)
+      : messages;
+
+  // Step 4: Process the response depending on whether streaming is requested
+  if (stream) {
+    return handleStreamResponse(
+      service,
+      adaptedMessages,
+      model,
+      maxTokens,
+      temperature,
+      systemPrompt,
+      tools,
+      provider
+    );
+  } else {
+    return handleNonStreamResponse(
+      service,
+      adaptedMessages,
+      model,
+      maxTokens,
+      temperature,
+      systemPrompt,
+      tools,
+      provider
+    );
+  }
+}
+
+// Helper for non-streaming response
+async function handleNonStreamResponse(
+  service: OpenAIService | AwsBedrockAnthropicService,
+  messages: Messages,
+  model: SupportedLLMs,
+  maxTokens: number,
+  temperature: number,
+  systemPrompt: string,
+  tools: any,
+  provider: Providers
+): Promise<LLMResponse> {
+  const response = await service.generateCompletion(
+    provider === Providers.OPENAI
+      ? (messages as OpenAIMessages)
+      : (messages as BedrockAnthropicMessages as any),
+    model,
+    maxTokens,
+    temperature,
+    systemPrompt,
+    tools
+  );
+
+  // Step 6: Adapt the response if provider is not OpenAI
+  return provider === Providers.OPENAI
+    ? response
+    : OutputFormatAdapter.adaptResponse(response, provider);
+}
+
+// Helper for streaming response
+async function* handleStreamResponse(
+  service: OpenAIService | AwsBedrockAnthropicService,
+  messages: Messages,
+  model: SupportedLLMs,
+  maxTokens: number,
+  temperature: number,
+  systemPrompt: string,
+  tools: any,
+  provider: Providers
+): AsyncGenerator<LLMResponse> {
+  const stream = service.generateStreamCompletion(
+    provider === Providers.OPENAI
+      ? (messages as OpenAIMessages)
+      : (messages as BedrockAnthropicMessages as any),
+    model,
+    maxTokens,
+    temperature,
+    systemPrompt,
+    tools,
+    true
+  );
+
+  // Step 7: Yield adapted chunks if the provider is not OpenAI
+  for await (const chunk of stream) {
+    yield provider === Providers.OPENAI
+      ? chunk
+      : OutputFormatAdapter.adaptResponse(chunk, provider);
+  }
+}

package/src/middleware/InputFormatAdapter.ts

@@ -0,0 +1,51 @@
+import {
+  BedrockAnthropicContentType,
+  BedrockAnthropicMessage,
+  BedrockAnthropicMessageRole,
+  Messages,
+  OpenAIFunctionMessage,
+  OpenAIMessages,
+  OpenAIMessagesRoles,
+  Providers,
+} from "../types";
+
+export class InputFormatAdapter {
+  static adaptMessages(
+    messages: Messages,
+    provider: Providers
+  ): OpenAIMessages | BedrockAnthropicMessage[] {
+    switch (provider) {
+      case Providers.OPENAI:
+        return messages.map((msg) => {
+          if (msg.role === OpenAIMessagesRoles.FUNCTION) {
+            return {
+              role: msg.role,
+              content: msg.content,
+              name: (msg as OpenAIFunctionMessage).name,
+            };
+          }
+          return {
+            role: msg.role,
+            content: msg.content as string,
+          };
+        }) as OpenAIMessages;
+
+      case Providers.ANTHROPIC_BEDROCK:
+        return messages.map((msg) => ({
+          role:
+            msg.role === OpenAIMessagesRoles.USER
+              ? BedrockAnthropicMessageRole.USER
+              : BedrockAnthropicMessageRole.ASSISTANT,
+          content: [
+            {
+              type: BedrockAnthropicContentType.TEXT,
+              text: msg.content as string,
+            },
+          ],
+        })) as BedrockAnthropicMessage[];
+
+      default:
+        throw new Error(`Unsupported provider: ${provider}`);
+    }
+  }
+}

package/src/middleware/OutputFormatAdapter.ts

@@ -0,0 +1,83 @@
+import {
+  BedrockAnthropicResponse,
+  OpenAIResponse,
+  LLMResponse,
+  Providers,
+  BedrockAnthropicContent,
+  BedrockAnthropicContentType,
+  BedrockAnthropicTextContent,
+  BedrockAnthropicToolResultContent,
+  BedrockAnthropicToolUseContent,
+} from "../types";
+
+export class OutputFormatAdapter {
+  static adaptResponse(response: any, provider: Providers): LLMResponse {
+    switch (provider) {
+      case Providers.OPENAI:
+        return response as OpenAIResponse;
+
+      case Providers.ANTHROPIC_BEDROCK:
+        return this.adaptAnthropicBedrockResponse(
+          response as BedrockAnthropicResponse
+        );
+
+      default:
+        throw new Error(`Unsupported provider: ${provider}`);
+    }
+  }
+
+  private static adaptAnthropicBedrockResponse(
+    response: BedrockAnthropicResponse
+  ): OpenAIResponse {
+    const openAIResponse: OpenAIResponse = {
+      id: response.id,
+      object: "text_completion",
+      created: Date.now(),
+      model: response.model,
+      choices: response.content.map((contentBlock, index) => ({
+        index,
+        message: {
+          role: this.mapRole(contentBlock),
+          content: this.extractContent(contentBlock),
+        },
+        logprobs: null,
+        finish_reason: response.stop_reason,
+      })),
+      usage: {
+        prompt_tokens: response.usage.input_tokens,
+        completion_tokens: response.usage.output_tokens,
+        total_tokens:
+          response.usage.input_tokens + response.usage.output_tokens,
+        prompt_tokens_details: { cached_tokens: 0 },
+        completion_tokens_details: { reasoning_tokens: 0 },
+      },
+      system_fingerprint: "anthropic_translation",
+    };
+    return openAIResponse;
+  }
+
+  private static mapRole(content: BedrockAnthropicContent): string {
+    switch (content.type) {
+      case BedrockAnthropicContentType.TOOL_USE:
+      case BedrockAnthropicContentType.TOOL_RESULT:
+        return "tool";
+      case BedrockAnthropicContentType.TEXT:
+        return "assistant";
+      default:
+        return "unknown";
+    }
+  }
+
+  private static extractContent(content: BedrockAnthropicContent): string {
+    switch (content.type) {
+      case BedrockAnthropicContentType.TEXT:
+        return (content as BedrockAnthropicTextContent).text;
+      case BedrockAnthropicContentType.TOOL_RESULT:
+        return (content as BedrockAnthropicToolResultContent).content || "";
+      case BedrockAnthropicContentType.TOOL_USE:
+        return (content as BedrockAnthropicToolUseContent).id || "";
+      default:
+        return "";
+    }
+  }
+}

package/src/middleware/ProviderFinder.ts

@@ -0,0 +1,23 @@
+import {
+  BedrockAnthropicSupportedLLMs,
+  OpenAISupportedLLMs,
+  Providers,
+  SupportedLLMs,
+} from "../types";
+
+export class ProviderFinder {
+  static getProvider(model: SupportedLLMs): Providers {
+    if (
+      model.type === "OpenAI" &&
+      Object.values(OpenAISupportedLLMs).includes(model.model)
+    ) {
+      return Providers.OPENAI;
+    } else if (
+      model.type === "BedrockAnthropic" &&
+      Object.values(BedrockAnthropicSupportedLLMs).includes(model.model)
+    ) {
+      return Providers.ANTHROPIC_BEDROCK;
+    }
+    throw new Error(`Unsupported model: ${model.model}`);
+  }
+}

package/src/services/AwsBedrockAnthropicService.ts

@@ -0,0 +1,111 @@
+import {
+  BedrockAnthropicParsedChunk,
+  BedrockAnthropicResponse,
+  Messages,
+  SupportedLLMs,
+} from "../types";
+import {
+  InvokeModelCommand,
+  BedrockRuntimeClient,
+  InvokeModelWithResponseStreamCommand,
+} from "@aws-sdk/client-bedrock-runtime";
+import { ClientService } from "./ClientService";
+
+export class AwsBedrockAnthropicService implements ClientService {
+  private bedrock: BedrockRuntimeClient;
+
+  constructor(awsAccessKey: string, awsSecretKey: string, region: string) {
+    this.bedrock = new BedrockRuntimeClient({
+      region,
+      credentials: {
+        accessKeyId: awsAccessKey,
+        secretAccessKey: awsSecretKey,
+      },
+    });
+  }
+
+  async generateCompletion(
+    messages: Messages,
+    model?: SupportedLLMs,
+    maxTokens?: number,
+    temperature?: number,
+    systemPrompt?: string,
+    tools?: any
+  ): Promise<BedrockAnthropicResponse> {
+    const modelId =
+      model?.type === "BedrockAnthropic" ? model.model : undefined;
+    if (!modelId) {
+      throw new Error("Invalid model type for AwsBedrockAnthropicService");
+    }
+
+    const body = JSON.stringify({
+      anthropic_version: "bedrock-2023-05-31",
+      max_tokens: maxTokens,
+      temperature,
+      messages,
+      system: systemPrompt,
+      ...(tools && tools.length > 0 ? { tools } : {}),
+    });
+
+    const command = new InvokeModelCommand({
+      modelId,
+      body,
+      contentType: "application/json",
+      accept: "application/json",
+    });
+
+    const response = await this.bedrock.send(command);
+    return JSON.parse(new TextDecoder().decode(response.body));
+  }
+
+  async *generateStreamCompletion(
+    messages: Messages,
+    model?: SupportedLLMs,
+    maxTokens?: number,
+    temperature?: number,
+    systemPrompt?: string,
+    tools?: any,
+    stream?: boolean
+  ): AsyncGenerator<BedrockAnthropicParsedChunk, void, unknown> {
+    const modelId =
+      model?.type === "BedrockAnthropic" ? model.model : undefined;
+    if (!modelId) {
+      throw new Error("Invalid model type for AwsBedrockAnthropicService");
+    }
+
+    const body = JSON.stringify({
+      anthropic_version: "bedrock-2023-05-31",
+      max_tokens: maxTokens,
+      temperature,
+      messages,
+      system: systemPrompt,
+      ...(tools && tools.length > 0 ? { tools } : {}),
+    });
+
+    const command = new InvokeModelWithResponseStreamCommand({
+      modelId,
+      body,
+      contentType: "application/json",
+      accept: "application/json",
+    });
+
+    const response = await this.bedrock.send(command);
+
+    if (response.body) {
+      const decoder = new TextDecoder("utf-8");
+
+      for await (const payload of response.body) {
+        const decodedString = decoder.decode(payload.chunk?.bytes, {
+          stream: true,
+        });
+
+        try {
+          const jsonObject = JSON.parse(decodedString);
+          yield jsonObject;
+        } catch (error) {
+          console.error("Failed to parse chunk as JSON:", error);
+        }
+      }
+    }
+  }
+}

package/src/services/ClientService.ts

@@ -0,0 +1,29 @@
+import {
+  BedrockAnthropicParsedChunk,
+  LLMResponse,
+  Messages,
+  SupportedLLMs,
+} from "../types";
+
+// for non streaming responses
+export interface ClientService {
+  generateCompletion(
+    messages: Messages,
+    model?: SupportedLLMs,
+    maxTokens?: number,
+    temperature?: number,
+    systemPrompt?: string,
+    tools?: any
+  ): Promise<LLMResponse>;
+
+  // For streaming responses
+  generateStreamCompletion(
+    messages: Messages,
+    model?: SupportedLLMs,
+    maxTokens?: number,
+    temperature?: number,
+    systemPrompt?: string,
+    tools?: any,
+    stream?: boolean
+  ): AsyncGenerator<BedrockAnthropicParsedChunk, void, unknown>;
+}

package/src/services/OpenAIService.ts

@@ -0,0 +1,68 @@
+import OpenAI from "openai";
+import { OpenAIMessages, OpenAIResponse, SupportedLLMs } from "../types";
+import { ClientService } from "./ClientService";
+
+export class OpenAIService implements ClientService {
+  private openai: OpenAI;
+
+  constructor(apiKey: string) {
+    this.openai = new OpenAI({ apiKey });
+  }
+
+  async generateCompletion(
+    messages: OpenAIMessages,
+    model: SupportedLLMs,
+    maxTokens: number,
+    temperature: number,
+    systemPrompt?: string, // Optional parameter
+    tools?: any // Optional parameter
+  ): Promise<OpenAIResponse> {
+    if (model.type !== "OpenAI") {
+      throw new Error("Unsupported model type for OpenAIService.");
+    }
+
+    try {
+      const response = await this.openai.chat.completions.create({
+        model: model.model,
+        messages,
+        max_tokens: maxTokens,
+        temperature,
+      });
+      return response as OpenAIResponse;
+    } catch (error) {
+      console.error("Error generating text:", error);
+      throw error;
+    }
+  }
+
+  async *generateStreamCompletion(
+    messages: OpenAIMessages,
+    model: SupportedLLMs,
+    maxTokens: number,
+    temperature: number,
+    systemPrompt?: string, // Optional parameter
+    tools?: any, // Optional parameter
+    stream?: boolean // Optional parameter
+  ): AsyncGenerator<any, void, unknown> {
+    if (model.type !== "OpenAI") {
+      throw new Error("Unsupported model type for OpenAIService.");
+    }
+
+    try {
+      const stream = await this.openai.chat.completions.create({
+        model: model.model,
+        messages,
+        max_tokens: maxTokens,
+        temperature,
+        stream: true,
+      });
+
+      for await (const chunk of stream) {
+        yield chunk;
+      }
+    } catch (error) {
+      console.error("Error in stream completion:", error);
+      throw error;
+    }
+  }
+}