bedrock-wrapper 1.0.10

package/.example.env

@@ -0,0 +1,13 @@
+# ========================
+# == AWS AUTH VARIABLES ==
+# ========================
+AWS_REGION = "us-west-2"
+AWS_ACCESS_KEY_ID = "AKxxxxxxxxxxxxxxxxxx"
+AWS_SECRET_ACCESS_KEY = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+
+# ================
+# == LLM PARAMS ==
+# ================
+LLM_MAX_GEN_TOKENS = 800
+LLM_TEMPERATURE = 0.1
+LLM_TOP_P = 0.9

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Justin Parker
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,119 @@
+# 🪨 Bedrock Wrapper
+Bedrock Wrapper is an npm package that simplifies the integration of existing OpenAI-compatible API objects with AWS Bedrock's serverless inference LLMs.  Follow the steps below to integrate into your own application, or alternativly use the 🪨 [Bedrock Proxy Endpoint](https://github.com/jparkerweb/bedrock-proxy-endpoint) project to spin up your own custom OpenAI server endpoint for even easier inference (using the standard `baseUrl`, and `apiKey` params).
+
+---
+
+### Install
+
+- install package: `npm install bedrock-wrapper`
+
+---
+
+### Usage
+
+1. import `bedrockWrapper`  
+    ```javascript
+    import { bedrockWrapper } from "bedrock-wrapper";
+    ```
+
+2. create an `awsCreds` object and fill in your AWS credentials  
+    ```javascript
+    const awsCreds = {
+        region: AWS_REGION,
+        accessKeyId: AWS_ACCESS_KEY_ID,
+        secretAccessKey: AWS_SECRET_ACCESS_KEY,
+    };
+    ```
+
+3. clone your openai chat completions object into `openaiChatCompletionsCreateObject` or create a new one and edit the values  
+    ```javascript
+    const openaiChatCompletionsCreateObject = {
+        "messages": messages,
+        "model": "Llama-3-8b",
+        "max_tokens": LLM_MAX_GEN_TOKENS,
+        "stream": true,
+        "temperature": LLM_TEMPERATURE,
+        "top_p": LLM_TOP_P,
+    };
+    ```
+
+    the `messages` variable should be in openai's role/content format  
+    ```javascript
+    messages = [
+        {
+            role: "system",
+            content: "You are a helpful AI assistant that follows instructions extremely well. Answer the user questions accurately. Think step by step before answering the question. You will get a $100 tip if you provide the correct answer.",
+        },
+        {
+            role: "user",
+            content: "Describe why openai api standard used by lots of serverless LLM api providers is better than aws bedrock invoke api offered by aws bedrock. Limit your response to five sentences.",
+        },
+        {
+            role: "assistant",
+            content: "",
+        },
+    ]
+    ```
+
+    ***the `model` value should be either a corresponding `modelName` or `modelId` for the supported `bedrock_models` (see the Supported Models section below)***
+
+4. call the `bedrockWrapper` function and pass in the previously defined `awsCreds` and `openaiChatCompletionsCreateObject` objects  
+    ```javascript
+    // create a variable to hold the complete response
+    let completeResponse = "";
+    // invoke the streamed bedrock api response
+    for await (const chunk of bedrockWrapper(awsCreds, openaiChatCompletionsCreateObject)) {
+        completeResponse += chunk;
+        // ---------------------------------------------------
+        // -- each chunk is streamed as it is received here --
+        // ---------------------------------------------------
+        process.stdout.write(chunk); // ⇠ do stuff with the streamed chunk
+    }
+    // console.log(`\n\completeResponse:\n${completeResponse}\n`); // ⇠ optional do stuff with the complete response returned from the API reguardless of stream or not
+    ```
+
+    if calling the unstreamed version you can call bedrockWrapper like this  
+    ```javascript
+    // create a variable to hold the complete response
+    let completeResponse = "";
+    // invoke the streamed bedrock api response
+    if (!openaiChatCompletionsCreateObject.stream){ // invoke the unstreamed bedrock api response
+        const response = await bedrockWrapper(awsCreds, openaiChatCompletionsCreateObject);
+        for await (const data of response) {
+            const jsonString = new TextDecoder().decode(data.body);
+            const jsonResponse = JSON.parse(jsonString);
+            completeResponse += jsonResponse.generation;
+        }
+        // ----------------------------------------------------
+        // -- unstreamed complete response is available here --
+        // ----------------------------------------------------
+        console.log(`\n\completeResponse:\n${completeResponse}\n`); // ⇠ do stuff with the complete response
+    }
+
+---
+
+### Supported Models
+
+| modelName      | modelId                            |
+|----------------|------------------------------------|
+| Llama-3-8b     | meta.llama3-8b-instruct-v1:0       |
+| Llama-3-70b    | meta.llama3-70b-instruct-v1:0      |
+| Mixtral-8x7b   | mistral.mixtral-8x7b-instruct-v0:1 |
+| Mistral-Large  | mistral.mistral-large-2402-v1:0    |
+
+To return the list progrmatically you can import and call `listBedrockWrapperSupportedModels`:  
+```javascript
+import { listBedrockWrapperSupportedModels } from 'bedrock-wrapper';
+console.log(`\nsupported models:\n${JSON.stringify(await listBedrockWrapperSupportedModels())}\n`);
+```
+
+Additional Bedrock model support can be added.  
+Please modify the `bedrock_models.js` file and submit a PR 🏆 or create an Issue.
+
+---
+
+### 📢 P.S.
+
+In case you missed it at the beginning of this doc, for an even easier setup, use the 🔀 [Bedrock Proxy Endpoint](https://github.com/jparkerweb/bedrock-proxy-endpoint) project to spin up your own custom OpenAI server endpoint (using the standard `baseUrl`, and `apiKey` params).
+
+![bedrock-proxy-endpoing](./docs/bedrock-proxy-endpoint.jpg)

package/bedrock-models.js

@@ -0,0 +1,99 @@
+// Description: This file contains the model configurations
+export const bedrock_models = [
+    {
+        // ================
+        // == Llama 3 8b ==
+        // ================
+        "modelName":                     "Llama-3-8b",
+        "modelId":                       "meta.llama3-8b-instruct-v1:0",
+        "bos_text":                      "<|begin_of_text|>",
+        "role_system_message_prefix":    "",
+        "role_system_message_suffix":    "",
+        "role_system_prefix":            "<|start_header_id|>",
+        "role_system_suffix":            "<|end_header_id|>",
+        "role_user_message_prefix":      "",
+        "role_user_message_suffix":      "",
+        "role_user_prefix":              "<|start_header_id|>",
+        "role_user_suffix":              "<|end_header_id|>",
+        "role_assistant_message_prefix": "",
+        "role_assistant_message_suffix": "",
+        "role_assistant_prefix":         "<|start_header_id|>",
+        "role_assistant_suffix":         "<|end_header_id|>",
+        "eom_text":                      "<|eot_id|>",
+        "display_role_names":            true,
+        "max_tokens_param_name":         "max_gen_len",
+        "response_chunk_element":        "generation",
+    },
+    {
+        // =================
+        // == Llama 3 70b ==
+        // =================
+        "modelName":                     "Llama-3-70b",
+        "modelId":                       "meta.llama3-70b-instruct-v1:0",
+        "bos_text":                      "<|begin_of_text|>",
+        "role_system_message_prefix":    "",
+        "role_system_message_suffix":    "",
+        "role_system_prefix":            "<|start_header_id|>",
+        "role_system_suffix":            "<|end_header_id|>",
+        "role_user_message_prefix":      "",
+        "role_user_message_suffix":      "",
+        "role_user_prefix":              "<|start_header_id|>",
+        "role_user_suffix":              "<|end_header_id|>",
+        "role_assistant_message_prefix": "",
+        "role_assistant_message_suffix": "",
+        "role_assistant_prefix":         "<|start_header_id|>",
+        "role_assistant_suffix":         "<|end_header_id|>",
+        "eom_text":                      "<|eot_id|>",
+        "display_role_names":            true,
+        "max_tokens_param_name":         "max_gen_len",
+        "response_chunk_element":        "generation",
+    },
+    {
+        // ==================
+        // == Mixtral-8x7b ==
+        // ==================
+        "modelName":                     "Mixtral-8x7b",
+        "modelId":                       "mistral.mixtral-8x7b-instruct-v0:1",
+        "bos_text":                      "<s>",
+        "role_system_message_prefix":    "",
+        "role_system_message_suffix":    "",
+        "role_system_prefix":            "",
+        "role_system_suffix":            "",
+        "role_user_message_prefix":      "[INST]",
+        "role_user_message_suffix":      "[/INST]",
+        "role_user_prefix":              "",
+        "role_user_suffix":              "",
+        "role_assistant_message_prefix": "",
+        "role_assistant_message_suffix": "",
+        "role_assistant_prefix":         "",
+        "role_assistant_suffix":         "",
+        "eom_text":                      "</s>",
+        "display_role_names":            false,
+        "max_tokens_param_name":         "max_tokens",
+        "response_chunk_element":        "outputs[0].text",
+    },
+    {
+        // ===================
+        // == Mistral Large ==
+        // ===================
+        "modelName":                     "Mistral-Large",
+        "modelId":                       "mistral.mistral-large-2402-v1:0",
+        "bos_text":                      "<s>",
+        "role_system_message_prefix":    "",
+        "role_system_message_suffix":    "",
+        "role_system_prefix":            "",
+        "role_system_suffix":            "",
+        "role_user_message_prefix":      "[INST]",
+        "role_user_message_suffix":      "[/INST]",
+        "role_user_prefix":              "",
+        "role_user_suffix":              "",
+        "role_assistant_message_prefix": "",
+        "role_assistant_message_suffix": "",
+        "role_assistant_prefix":         "",
+        "role_assistant_suffix":         "",
+        "eom_text":                      "</s>",
+        "display_role_names":            false,
+        "max_tokens_param_name":         "max_tokens",
+        "response_chunk_element":        "outputs[0].text",
+    },
+];

package/bedrock-wrapper.js

@@ -0,0 +1,164 @@
+// ======================================================================
+// == 🪨 Bedrock Wrapper                                                ==
+// ==                                                                  ==
+// == Bedrock Wrapper is an npm package that simplifies the integration ==
+// == of existing OpenAI-compatible API objects AWS Bedrock's          ==
+// == serverless inference LLMs.                                       ==
+// ======================================================================
+writeAsciiArt();
+
+// -------------
+// -- imports --
+// -------------
+import { bedrock_models } from "./bedrock-models.js";
+import {
+    BedrockRuntimeClient,
+    InvokeModelCommand, InvokeModelWithResponseStreamCommand,
+} from "@aws-sdk/client-bedrock-runtime";
+
+// -------------------
+// -- main function --
+// -------------------
+export async function* bedrockWrapper(awsCreds, openaiChatCompletionsCreateObject, { logging = false } = {} ) {
+    const { region, accessKeyId, secretAccessKey } = awsCreds;
+    const { messages, model, max_tokens, stream, temperature, top_p } = openaiChatCompletionsCreateObject;
+
+
+    // retrieve the model configuration
+    const awsModel = bedrock_models.find((x) => (x.modelName.toLowerCase() === model.toLowerCase() || x.modelId.toLowerCase() === model.toLowerCase()));
+    if (!awsModel) { throw new Error(`Model configuration not found for model: ${model}`); }
+
+    // cleanup message content before formatting prompt message
+    let message_cleaned = [];
+    for (let i = 0; i < messages.length; i++) {
+        if (messages[i].content !== "") {
+            message_cleaned.push(messages[i]);
+        } else if (awsModel.display_role_names) {
+            message_cleaned.push(messages[i]);
+        }
+
+        if (i === (messages.length - 1) && messages[i].content !== "" && awsModel.display_role_names) {
+            message_cleaned.push({role: "assistant", content: ""});
+        }
+    }
+
+    // format prompt message from message array
+    let prompt = awsModel.bos_text;
+    let eom_text_inserted = false;
+    for (let i = 0; i < message_cleaned.length; i++) {
+        prompt += "\n";
+        if (message_cleaned[i].role === "system") {
+            prompt += awsModel.role_system_message_prefix;
+            prompt += awsModel.role_system_prefix;
+            if (awsModel.display_role_names) { prompt += message_cleaned[i].role; }
+            prompt += awsModel.role_system_suffix;
+            if (awsModel.display_role_names) {prompt += "\n"; }
+            prompt += message_cleaned[i].content;
+            prompt += awsModel.role_system_message_suffix;
+        } else if (message_cleaned[i].role === "user") {
+            prompt += awsModel.role_user_message_prefix;
+            prompt += awsModel.role_user_prefix;
+            if (awsModel.display_role_names) { prompt += message_cleaned[i].role; }
+            prompt += awsModel.role_user_suffix;
+            if (awsModel.display_role_names) {prompt += "\n"; }
+            prompt += message_cleaned[i].content;
+            prompt += awsModel.role_user_message_suffix;
+        } else if (message_cleaned[i].role === "assistant") {
+            prompt += awsModel.role_assistant_message_prefix;
+            prompt += awsModel.role_assistant_prefix;
+            if (awsModel.display_role_names) { prompt += message_cleaned[i].role; }
+            prompt += awsModel.role_assistant_suffix;
+            if (awsModel.display_role_names) {prompt += "\n"; }
+            prompt += message_cleaned[i].content;
+            prompt += awsModel.role_assistant_message_suffix;
+        }
+        if (message_cleaned[i+1] && message_cleaned[i+1].content === "") {
+            prompt += `\n${awsModel.eom_text}`;
+            eom_text_inserted = true;
+        } else if ((i+1) === (message_cleaned.length - 1) && !eom_text_inserted) {
+            prompt += `\n${awsModel.eom_text}`;
+        }
+    }
+    
+    // logging
+    if (logging) {
+        console.log(`\nPrompt: ${prompt}\n`);
+    }
+
+    // Format the request payload using the model's native structure.
+    const request = {
+        prompt,
+        // Optional inference parameters:
+        [awsModel.max_tokens_param_name]: max_tokens,
+        temperature: temperature,
+        top_p: top_p,
+    };
+    
+    // Create a Bedrock Runtime client in the AWS Region of your choice
+    const client = new BedrockRuntimeClient({
+        region: region,
+        credentials: {
+            accessKeyId: accessKeyId,
+            secretAccessKey: secretAccessKey,
+        },
+    });
+
+    if (stream) {
+        const responseStream = await client.send(
+            new InvokeModelWithResponseStreamCommand({
+                contentType: "application/json",
+                body: JSON.stringify(request),
+                modelId: awsModel.modelId,
+            }),
+        );
+        for await (const event of responseStream.body) {
+            const chunk = JSON.parse(new TextDecoder().decode(event.chunk.bytes));
+            let result = getValueByPath(chunk, awsModel.response_chunk_element);
+            if (result) {
+                yield result;
+            }
+        }
+    } else {
+        const apiResponse = await client.send(
+            new InvokeModelCommand({
+              contentType: "application/json",
+              body: JSON.stringify(request),
+              modelId: awsModel.modelId,
+            }),
+          );
+        yield apiResponse;
+    }    
+}
+
+
+// ---------------------------
+// -- list supported models --
+// ---------------------------
+export async function listBedrockWrapperSupportedModels() {
+    let supported_models = [];
+    for (let i = 0; i < bedrock_models.length; i++) {
+        supported_models.push(`{"modelName": ${bedrock_models[i].modelName}, "modelId": ${bedrock_models[i].modelId}}`);
+    }
+    return supported_models;
+}
+
+
+// ----------------------
+// -- helper functions --
+// ----------------------
+// helper function to get a value from an object using a path string
+function getValueByPath(obj, path) {
+    // Split the path into an array of keys
+    let keys = path.replace(/\[(\w+)\]/g, '.$1').split('.');  // Convert indexes into properties
+    // Reduce the keys array to the final value
+    return keys.reduce((acc, key) => acc && acc[key], obj);
+}
+// helper function to write ascii art
+function writeAsciiArt() {
+    console.log(`
+     ___         _                 _     ___                     _ 
+    | . > ___  _| | _ _  ___  ___ | |__ |_ _|_ _ ._ _ ._ _  ___ | |
+    | . \\/ ._>/ . || '_>/ . \\/ | '| / /  | || | || ' || ' |/ ._>| |
+    |___/\\___.\\___||_|  \\___/\\_|_.|_\\_\\  |_|\`___||_|_||_|_|\\___.|_|
+    `);
+}

package/example.js

@@ -0,0 +1,95 @@
+// ================================================================================
+// == AWS Bedrock Example: Invoke a Model with a Streamed or Unstreamed Response ==
+// ================================================================================
+
+// ---------------------------------------------------------------------
+// -- import environment variables from .env file or define them here --
+// ---------------------------------------------------------------------
+import dotenv from 'dotenv';
+dotenv.config();
+const AWS_REGION = process.env.AWS_REGION;
+const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID;
+const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY;
+const LLM_MAX_GEN_TOKENS = parseInt(process.env.LLM_MAX_GEN_TOKENS);
+const LLM_TEMPERATURE = parseFloat(process.env.LLM_TEMPERATURE);
+const LLM_TOP_P = parseFloat(process.env.LLM_TOP_P);
+
+// --------------------------------------------
+// -- import functions from bedrock-wrapper   --
+// --     - bedrockWrapper                 --
+// --     - listBedrockWrapperSupportedModels --
+// --------------------------------------------
+import { bedrockWrapper, listBedrockWrapperSupportedModels } from "bedrock-wrapper";
+
+// ----------------------------------------------
+// -- example call to list of supported models --
+// ----------------------------------------------
+console.log(`\nsupported models:\n${JSON.stringify(await listBedrockWrapperSupportedModels())}\n`);
+
+// -----------------------------------------------
+// -- example prompt in `messages` array format --
+// -----------------------------------------------
+const messages = [
+    {
+        role: "system",
+        content: "You are a helpful AI assistant that follows instructions extremely well. Answer the user questions accurately. Think step by step before answering the question. You will get a $100 tip if you provide the correct answer.",
+    },
+    {
+        role: "user",
+        content: "Describe why openai api standard used by lots of serverless LLM api providers is better than aws bedrock invoke api offered by aws bedrock. Limit your response to five sentences.",
+    },
+    {
+        role: "assistant",
+        content: "",
+    },
+];
+
+
+// ---------------------------------------------------
+// -- create an object to hold your AWS credentials --
+// ---------------------------------------------------
+const awsCreds = {
+    region: AWS_REGION,
+    accessKeyId: AWS_ACCESS_KEY_ID,
+    secretAccessKey: AWS_SECRET_ACCESS_KEY,
+};
+// ----------------------------------------------------------------------
+// -- create an object that copies your openai chat completions object --
+// ----------------------------------------------------------------------
+const openaiChatCompletionsCreateObject = {
+    "messages": messages,
+    "model": "Llama-3-8b",
+    "max_tokens": LLM_MAX_GEN_TOKENS,
+    "stream": true,
+    "temperature": LLM_TEMPERATURE,
+    "top_p": LLM_TOP_P,
+};
+
+
+// ------------------------------------------------------------
+// -- invoke the streamed or unstreamed bedrock api response --
+// ------------------------------------------------------------
+// create a variable to hold the complete response
+let completeResponse = "";
+// streamed call
+if (openaiChatCompletionsCreateObject.stream) {
+    for await (const chunk of bedrockWrapper(awsCreds, openaiChatCompletionsCreateObject, { logging:true })) {
+        completeResponse += chunk;
+        // ---------------------------------------------------
+        // -- each chunk is streamed as it is received here --
+        // ---------------------------------------------------
+        process.stdout.write(chunk); // ⇠ do stuff with the streamed chunk
+    }
+} else { // unstreamed call
+    const response = await bedrockWrapper(awsCreds, openaiChatCompletionsCreateObject, { logging:true });
+    for await (const data of response) {
+        const jsonString = new TextDecoder().decode(data.body);
+        const jsonResponse = JSON.parse(jsonString);
+        completeResponse += jsonResponse.generation;
+    }
+    // ----------------------------------------------------
+    // -- unstreamed complete response is available here --
+    // ----------------------------------------------------
+    console.log(`\n\completeResponse:\n${completeResponse}\n`); // ⇠ do stuff with the complete response
+}
+// console.log(`\n\completeResponse:\n${completeResponse}\n`); // ⇠ optional do stuff with the complete response returned from the API reguardless of stream or not

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "bedrock-wrapper",
+  "version": "1.0.10",
+  "description": "🪨 Bedrock Wrapper is an npm package that simplifies the integration of existing OpenAI-compatible API objects with AWS Bedrock's serverless inference LLMs.",
+  "repository": {
+		"type": "git",
+		"url": "https://github.com/jparkerweb/bedrock-wrapper.git"
+	},
+  "main": "aws-bedrock-wrapper.js",
+  "type": "module",
+  "scripts": {
+    "example": "node example.js"
+  },
+  "keywords": [
+    "openai",
+    "bedrock",
+    "aws",
+    "serverless",
+    "inference",
+    "llm"
+  ],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@aws-sdk/client-bedrock-runtime": "^3.567.0",
+    "dotenv": "^16.4.5"
+  }
+}