bedrock-wrapper 1.0.15 → 1.2.0

package/README.md

@@ -29,7 +29,7 @@ Bedrock Wrapper is an npm package that simplifies the integration of existing Op
     ```javascript
     const openaiChatCompletionsCreateObject = {
         "messages": messages,
-        "model": "Llama-3-8b",
+        "model": "Llama-3-1-8b",
         "max_tokens": LLM_MAX_GEN_TOKENS,
         "stream": true,
         "temperature": LLM_TEMPERATURE,
@@ -96,6 +96,9 @@ Bedrock Wrapper is an npm package that simplifies the integration of existing Op
 
 | modelName      | modelId                            |
 |----------------|------------------------------------|
+| Llama-3-1-8b   | meta.llama3-1-8b-instruct-v1:0     |
+| Llama-3-1-70b  | meta.llama3-1-70b-instruct-v1:0    |
+| Llama-3-1-405b | meta.llama3-1-405b-instruct-v1:0   |
 | Llama-3-8b     | meta.llama3-8b-instruct-v1:0       |
 | Llama-3-70b    | meta.llama3-70b-instruct-v1:0      |
 | Mistral-7b     | mistral.mistral-7b-instruct-v0:2   |

package/bedrock-models.js

@@ -1,5 +1,80 @@
 // Description: This file contains the model configurations
 export const bedrock_models = [
+    {
+        // ==================
+        // == Llama 3.1 8b ==
+        // ==================
+        "modelName":                     "Llama-3-1-8b",
+        "modelId":                       "meta.llama3-1-8b-instruct-v1:0",
+        "bos_text":                      "<|begin_of_text|>",
+        "role_system_message_prefix":    "",
+        "role_system_message_suffix":    "",
+        "role_system_prefix":            "<|start_header_id|>",
+        "role_system_suffix":            "<|end_header_id|>",
+        "role_user_message_prefix":      "",
+        "role_user_message_suffix":      "",
+        "role_user_prefix":              "<|start_header_id|>",
+        "role_user_suffix":              "<|end_header_id|>",
+        "role_assistant_message_prefix": "",
+        "role_assistant_message_suffix": "",
+        "role_assistant_prefix":         "<|start_header_id|>",
+        "role_assistant_suffix":         "<|end_header_id|>",
+        "eom_text":                      "<|eot_id|>",
+        "display_role_names":            true,
+        "max_tokens_param_name":         "max_gen_len",
+        "max_supported_response_tokens": 2048,
+        "response_chunk_element":        "generation",
+    },
+    {
+        // ===================
+        // == Llama 3.1 70b ==
+        // ===================
+        "modelName":                     "Llama-3-1-70b",
+        "modelId":                       "meta.llama3-1-70b-instruct-v1:0",
+        "bos_text":                      "<|begin_of_text|>",
+        "role_system_message_prefix":    "",
+        "role_system_message_suffix":    "",
+        "role_system_prefix":            "<|start_header_id|>",
+        "role_system_suffix":            "<|end_header_id|>",
+        "role_user_message_prefix":      "",
+        "role_user_message_suffix":      "",
+        "role_user_prefix":              "<|start_header_id|>",
+        "role_user_suffix":              "<|end_header_id|>",
+        "role_assistant_message_prefix": "",
+        "role_assistant_message_suffix": "",
+        "role_assistant_prefix":         "<|start_header_id|>",
+        "role_assistant_suffix":         "<|end_header_id|>",
+        "eom_text":                      "<|eot_id|>",
+        "display_role_names":            true,
+        "max_tokens_param_name":         "max_gen_len",
+        "max_supported_response_tokens": 2048,
+        "response_chunk_element":        "generation",
+    },
+    {
+        // ====================
+        // == Llama 3.1 405b ==
+        // ====================
+        "modelName":                     "Llama-3-1-405b",
+        "modelId":                       "meta.llama3-1-405b-instruct-v1:0",
+        "bos_text":                      "<|begin_of_text|>",
+        "role_system_message_prefix":    "",
+        "role_system_message_suffix":    "",
+        "role_system_prefix":            "<|start_header_id|>",
+        "role_system_suffix":            "<|end_header_id|>",
+        "role_user_message_prefix":      "",
+        "role_user_message_suffix":      "",
+        "role_user_prefix":              "<|start_header_id|>",
+        "role_user_suffix":              "<|end_header_id|>",
+        "role_assistant_message_prefix": "",
+        "role_assistant_message_suffix": "",
+        "role_assistant_prefix":         "<|start_header_id|>",
+        "role_assistant_suffix":         "<|end_header_id|>",
+        "eom_text":                      "<|eot_id|>",
+        "display_role_names":            true,
+        "max_tokens_param_name":         "max_gen_len",
+        "max_supported_response_tokens": 2048,
+        "response_chunk_element":        "generation",
+    },
     {
         // ================
         // == Llama 3 8b ==

package/bedrock-wrapper.js

@@ -1,156 +1,156 @@
-// ======================================================================
-// == 🪨 Bedrock Wrapper                                                ==
-// ==                                                                  ==
-// == Bedrock Wrapper is an npm package that simplifies the integration ==
-// == of existing OpenAI-compatible API objects AWS Bedrock's          ==
-// == serverless inference LLMs.                                       ==
-// ======================================================================
-
-// -------------
-// -- imports --
-// -------------
-// Bedrock model configurations
-import { bedrock_models } from "./bedrock-models.js";
-// AWS SDK
-import {
-    BedrockRuntimeClient,
-    InvokeModelCommand, InvokeModelWithResponseStreamCommand,
-} from "@aws-sdk/client-bedrock-runtime";
-// helper functions
-import {
-    getValueByPath,
-    writeAsciiArt
-} from "./utils.js";
-
-
-// write the ascii art logo on initial load
-writeAsciiArt();
-
-
-// -------------------
-// -- main function --
-// -------------------
-export async function* bedrockWrapper(awsCreds, openaiChatCompletionsCreateObject, { logging = false } = {} ) {
-    const { region, accessKeyId, secretAccessKey } = awsCreds;
-    const { messages, model, max_tokens, stream, temperature, top_p } = openaiChatCompletionsCreateObject;
-
-
-    // retrieve the model configuration
-    const awsModel = bedrock_models.find((x) => (x.modelName.toLowerCase() === model.toLowerCase() || x.modelId.toLowerCase() === model.toLowerCase()));
-    if (!awsModel) { throw new Error(`Model configuration not found for model: ${model}`); }
-
-    // cleanup message content before formatting prompt message
-    let message_cleaned = [];
-    for (let i = 0; i < messages.length; i++) {
-        if (messages[i].content !== "") {
-            message_cleaned.push(messages[i]);
-        } else if (awsModel.display_role_names) {
-            message_cleaned.push(messages[i]);
-        }
-
-        if (i === (messages.length - 1) && messages[i].content !== "" && awsModel.display_role_names) {
-            message_cleaned.push({role: "assistant", content: ""});
-        }
-    }
-
-    // format prompt message from message array
-    let prompt = awsModel.bos_text;
-    let eom_text_inserted = false;
-    for (let i = 0; i < message_cleaned.length; i++) {
-        prompt += "\n";
-        if (message_cleaned[i].role === "system") {
-            prompt += awsModel.role_system_message_prefix;
-            prompt += awsModel.role_system_prefix;
-            if (awsModel.display_role_names) { prompt += message_cleaned[i].role; }
-            prompt += awsModel.role_system_suffix;
-            if (awsModel.display_role_names) {prompt += "\n"; }
-            prompt += message_cleaned[i].content;
-            prompt += awsModel.role_system_message_suffix;
-        } else if (message_cleaned[i].role === "user") {
-            prompt += awsModel.role_user_message_prefix;
-            prompt += awsModel.role_user_prefix;
-            if (awsModel.display_role_names) { prompt += message_cleaned[i].role; }
-            prompt += awsModel.role_user_suffix;
-            if (awsModel.display_role_names) {prompt += "\n"; }
-            prompt += message_cleaned[i].content;
-            prompt += awsModel.role_user_message_suffix;
-        } else if (message_cleaned[i].role === "assistant") {
-            prompt += awsModel.role_assistant_message_prefix;
-            prompt += awsModel.role_assistant_prefix;
-            if (awsModel.display_role_names) { prompt += message_cleaned[i].role; }
-            prompt += awsModel.role_assistant_suffix;
-            if (awsModel.display_role_names) {prompt += "\n"; }
-            prompt += message_cleaned[i].content;
-            prompt += awsModel.role_assistant_message_suffix;
-        }
-        if (message_cleaned[i+1] && message_cleaned[i+1].content === "") {
-            prompt += `\n${awsModel.eom_text}`;
-            eom_text_inserted = true;
-        } else if ((i+1) === (message_cleaned.length - 1) && !eom_text_inserted) {
-            prompt += `\n${awsModel.eom_text}`;
-        }
-    }
-    
-    // logging
-    if (logging) {
-        console.log(`\nPrompt: ${prompt}\n`);
-    }
-
-    const max_gen_tokens = max_tokens <= awsModel.max_supported_response_tokens ? max_tokens : awsModel.max_supported_response_tokens;
-
-    // Format the request payload using the model's native structure.
-    const request = {
-        prompt,
-        // Optional inference parameters:
-        [awsModel.max_tokens_param_name]: max_gen_tokens,
-        temperature: temperature,
-        top_p: top_p,
-    };
-    
-    // Create a Bedrock Runtime client in the AWS Region of your choice
-    const client = new BedrockRuntimeClient({
-        region: region,
-        credentials: {
-            accessKeyId: accessKeyId,
-            secretAccessKey: secretAccessKey,
-        },
-    });
-
-    if (stream) {
-        const responseStream = await client.send(
-            new InvokeModelWithResponseStreamCommand({
-                contentType: "application/json",
-                body: JSON.stringify(request),
-                modelId: awsModel.modelId,
-            }),
-        );
-        for await (const event of responseStream.body) {
-            const chunk = JSON.parse(new TextDecoder().decode(event.chunk.bytes));
-            let result = getValueByPath(chunk, awsModel.response_chunk_element);
-            if (result) {
-                yield result;
-            }
-        }
-    } else {
-        const apiResponse = await client.send(
-            new InvokeModelCommand({
-              contentType: "application/json",
-              body: JSON.stringify(request),
-              modelId: awsModel.modelId,
-            }),
-          );
-        yield apiResponse;
-    }    
-}
-
-
-// ---------------------------
-// -- list supported models --
-// ---------------------------
-export async function listBedrockWrapperSupportedModels() {
-    let supported_models = [];
-    for (let i = 0; i < bedrock_models.length; i++) {
-        supported_models.push(`{"modelName": ${bedrock_models[i].modelName}, "modelId": ${bedrock_models[i].modelId}}`);
-    }
-    return supported_models;
-}
+// ======================================================================
+// == 🪨 Bedrock Wrapper                                                ==
+// ==                                                                  ==
+// == Bedrock Wrapper is an npm package that simplifies the integration ==
+// == of existing OpenAI-compatible API objects AWS Bedrock's          ==
+// == serverless inference LLMs.                                       ==
+// ======================================================================
+
+// -------------
+// -- imports --
+// -------------
+// Bedrock model configurations
+import { bedrock_models } from "./bedrock-models.js";
+// AWS SDK
+import {
+    BedrockRuntimeClient,
+    InvokeModelCommand, InvokeModelWithResponseStreamCommand,
+} from "@aws-sdk/client-bedrock-runtime";
+// helper functions
+import {
+    getValueByPath,
+    writeAsciiArt
+} from "./utils.js";
+
+
+// write the ascii art logo on initial load
+writeAsciiArt();
+
+
+// -------------------
+// -- main function --
+// -------------------
+export async function* bedrockWrapper(awsCreds, openaiChatCompletionsCreateObject, { logging = false } = {} ) {
+    const { region, accessKeyId, secretAccessKey } = awsCreds;
+    const { messages, model, max_tokens, stream, temperature, top_p } = openaiChatCompletionsCreateObject;
+
+
+    // retrieve the model configuration
+    const awsModel = bedrock_models.find((x) => (x.modelName.toLowerCase() === model.toLowerCase() || x.modelId.toLowerCase() === model.toLowerCase()));
+    if (!awsModel) { throw new Error(`Model configuration not found for model: ${model}`); }
+
+    // cleanup message content before formatting prompt message
+    let message_cleaned = [];
+    for (let i = 0; i < messages.length; i++) {
+        if (messages[i].content !== "") {
+            message_cleaned.push(messages[i]);
+        } else if (awsModel.display_role_names) {
+            message_cleaned.push(messages[i]);
+        }
+
+        if (i === (messages.length - 1) && messages[i].content !== "" && awsModel.display_role_names) {
+            message_cleaned.push({role: "assistant", content: ""});
+        }
+    }
+
+    // format prompt message from message array
+    let prompt = awsModel.bos_text;
+    let eom_text_inserted = false;
+    for (let i = 0; i < message_cleaned.length; i++) {
+        prompt += "\n";
+        if (message_cleaned[i].role === "system") {
+            prompt += awsModel.role_system_message_prefix;
+            prompt += awsModel.role_system_prefix;
+            if (awsModel.display_role_names) { prompt += message_cleaned[i].role; }
+            prompt += awsModel.role_system_suffix;
+            if (awsModel.display_role_names) {prompt += "\n"; }
+            prompt += message_cleaned[i].content;
+            prompt += awsModel.role_system_message_suffix;
+        } else if (message_cleaned[i].role === "user") {
+            prompt += awsModel.role_user_message_prefix;
+            prompt += awsModel.role_user_prefix;
+            if (awsModel.display_role_names) { prompt += message_cleaned[i].role; }
+            prompt += awsModel.role_user_suffix;
+            if (awsModel.display_role_names) {prompt += "\n"; }
+            prompt += message_cleaned[i].content;
+            prompt += awsModel.role_user_message_suffix;
+        } else if (message_cleaned[i].role === "assistant") {
+            prompt += awsModel.role_assistant_message_prefix;
+            prompt += awsModel.role_assistant_prefix;
+            if (awsModel.display_role_names) { prompt += message_cleaned[i].role; }
+            prompt += awsModel.role_assistant_suffix;
+            if (awsModel.display_role_names) {prompt += "\n"; }
+            prompt += message_cleaned[i].content;
+            prompt += awsModel.role_assistant_message_suffix;
+        }
+        if (message_cleaned[i+1] && message_cleaned[i+1].content === "") {
+            prompt += `\n${awsModel.eom_text}`;
+            eom_text_inserted = true;
+        } else if ((i+1) === (message_cleaned.length - 1) && !eom_text_inserted) {
+            prompt += `\n${awsModel.eom_text}`;
+        }
+    }
+    
+    // logging
+    if (logging) {
+        console.log(`\nPrompt: ${prompt}\n`);
+    }
+
+    const max_gen_tokens = max_tokens <= awsModel.max_supported_response_tokens ? max_tokens : awsModel.max_supported_response_tokens;
+
+    // Format the request payload using the model's native structure.
+    const request = {
+        prompt,
+        // Optional inference parameters:
+        [awsModel.max_tokens_param_name]: max_gen_tokens,
+        temperature: temperature,
+        top_p: top_p,
+    };
+    
+    // Create a Bedrock Runtime client in the AWS Region of your choice
+    const client = new BedrockRuntimeClient({
+        region: region,
+        credentials: {
+            accessKeyId: accessKeyId,
+            secretAccessKey: secretAccessKey,
+        },
+    });
+
+    if (stream) {
+        const responseStream = await client.send(
+            new InvokeModelWithResponseStreamCommand({
+                contentType: "application/json",
+                body: JSON.stringify(request),
+                modelId: awsModel.modelId,
+            }),
+        );
+        for await (const event of responseStream.body) {
+            const chunk = JSON.parse(new TextDecoder().decode(event.chunk.bytes));
+            let result = getValueByPath(chunk, awsModel.response_chunk_element);
+            if (result) {
+                yield result;
+            }
+        }
+    } else {
+        const apiResponse = await client.send(
+            new InvokeModelCommand({
+              contentType: "application/json",
+              body: JSON.stringify(request),
+              modelId: awsModel.modelId,
+            }),
+          );
+        yield apiResponse;
+    }    
+}
+
+
+// ---------------------------
+// -- list supported models --
+// ---------------------------
+export async function listBedrockWrapperSupportedModels() {
+    let supported_models = [];
+    for (let i = 0; i < bedrock_models.length; i++) {
+        supported_models.push(`{"modelName": ${bedrock_models[i].modelName}, "modelId": ${bedrock_models[i].modelId}}`);
+    }
+    return supported_models;
+}

package/{example.js → example-test.js}

@@ -22,7 +22,7 @@ const LLM_TOP_P = parseFloat(process.env.LLM_TOP_P);
 import {
     bedrockWrapper,
     listBedrockWrapperSupportedModels
-} from "bedrock-wrapper";
+} from "./bedrock-wrapper.js";
 
 // ----------------------------------------------
 // -- example call to list of supported models --
@@ -35,11 +35,11 @@ console.log(`\nsupported models:\n${JSON.stringify(await listBedrockWrapperSuppo
 const messages = [
     {
         role: "system",
-        content: "You are a helpful AI assistant that follows instructions extremely well. Answer the user questions accurately. Think step by step before answering the question. You will get a $100 tip if you provide the correct answer.",
+        content: "You are a helpful AI assistant that follows instructions extremely well. Answer the user questions accurately. Think step by step before answering the question.",
     },
     {
         role: "user",
-        content: "Describe why openai api standard used by lots of serverless LLM api providers is better than aws bedrock invoke api offered by aws bedrock. Limit your response to five sentences.",
+        content: "Describe what the openai api standard used by lots of serverless LLM api providers is and why it has been widely adopted.",
     },
     {
         role: "assistant",
@@ -61,7 +61,7 @@ const awsCreds = {
 // ----------------------------------------------------------------------
 const openaiChatCompletionsCreateObject = {
     "messages": messages,
-    "model": "Llama-3-8b",
+    "model": "Llama-3-1-405b",
     "max_tokens": LLM_MAX_GEN_TOKENS,
     "stream": true,
     "temperature": LLM_TEMPERATURE,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bedrock-wrapper",
-  "version": "1.0.15",
+  "version": "1.2.0",
   "description": "🪨 Bedrock Wrapper is an npm package that simplifies the integration of existing OpenAI-compatible API objects with AWS Bedrock's serverless inference LLMs.",
   "repository": {
 		"type": "git",
@@ -21,7 +21,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@aws-sdk/client-bedrock-runtime": "^3.575.0",
+    "@aws-sdk/client-bedrock-runtime": "^3.623.0",
     "dotenv": "^16.4.5"
   }
 }