bedrock-wrapper 1.0.14 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -8
- package/bedrock-models.js +55 -0
- package/bedrock-wrapper.js +3 -1
- package/example.js +3 -3
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -29,7 +29,7 @@ Bedrock Wrapper is an npm package that simplifies the integration of existing Op
|
|
|
29
29
|
```javascript
|
|
30
30
|
const openaiChatCompletionsCreateObject = {
|
|
31
31
|
"messages": messages,
|
|
32
|
-
"model": "Llama-3-8b",
|
|
32
|
+
"model": "Llama-3-1-8b",
|
|
33
33
|
"max_tokens": LLM_MAX_GEN_TOKENS,
|
|
34
34
|
"stream": true,
|
|
35
35
|
"temperature": LLM_TEMPERATURE,
|
|
@@ -94,13 +94,15 @@ Bedrock Wrapper is an npm package that simplifies the integration of existing Op
|
|
|
94
94
|
|
|
95
95
|
### Supported Models
|
|
96
96
|
|
|
97
|
-
| modelName
|
|
98
|
-
|
|
99
|
-
| Llama-3-8b
|
|
100
|
-
| Llama-3-70b
|
|
101
|
-
|
|
|
102
|
-
|
|
|
103
|
-
| Mistral-
|
|
97
|
+
| modelName | modelId |
|
|
98
|
+
|---------------|------------------------------------|
|
|
99
|
+
| Llama-3-1-8b | meta.llama3-1-8b-instruct-v1:0 |
|
|
100
|
+
| Llama-3-1-70b | meta.llama3-1-70b-instruct-v1:0 |
|
|
101
|
+
| Llama-3-8b | meta.llama3-8b-instruct-v1:0 |
|
|
102
|
+
| Llama-3-70b | meta.llama3-70b-instruct-v1:0 |
|
|
103
|
+
| Mistral-7b | mistral.mistral-7b-instruct-v0:2 |
|
|
104
|
+
| Mixtral-8x7b | mistral.mixtral-8x7b-instruct-v0:1 |
|
|
105
|
+
| Mistral-Large | mistral.mistral-large-2402-v1:0 |
|
|
104
106
|
|
|
105
107
|
To return the list progrmatically you can import and call `listBedrockWrapperSupportedModels`:
|
|
106
108
|
```javascript
|
package/bedrock-models.js
CHANGED
|
@@ -1,5 +1,55 @@
|
|
|
1
1
|
// Description: This file contains the model configurations
|
|
2
2
|
export const bedrock_models = [
|
|
3
|
+
{
|
|
4
|
+
// ==================
|
|
5
|
+
// == Llama 3.1 8b ==
|
|
6
|
+
// ==================
|
|
7
|
+
"modelName": "Llama-3-1-8b",
|
|
8
|
+
"modelId": "meta.llama3-1-8b-instruct-v1:0",
|
|
9
|
+
"bos_text": "<|begin_of_text|>",
|
|
10
|
+
"role_system_message_prefix": "",
|
|
11
|
+
"role_system_message_suffix": "",
|
|
12
|
+
"role_system_prefix": "<|start_header_id|>",
|
|
13
|
+
"role_system_suffix": "<|end_header_id|>",
|
|
14
|
+
"role_user_message_prefix": "",
|
|
15
|
+
"role_user_message_suffix": "",
|
|
16
|
+
"role_user_prefix": "<|start_header_id|>",
|
|
17
|
+
"role_user_suffix": "<|end_header_id|>",
|
|
18
|
+
"role_assistant_message_prefix": "",
|
|
19
|
+
"role_assistant_message_suffix": "",
|
|
20
|
+
"role_assistant_prefix": "<|start_header_id|>",
|
|
21
|
+
"role_assistant_suffix": "<|end_header_id|>",
|
|
22
|
+
"eom_text": "<|eot_id|>",
|
|
23
|
+
"display_role_names": true,
|
|
24
|
+
"max_tokens_param_name": "max_gen_len",
|
|
25
|
+
"max_supported_response_tokens": 2048,
|
|
26
|
+
"response_chunk_element": "generation",
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
// ===================
|
|
30
|
+
// == Llama 3.1 70b ==
|
|
31
|
+
// ===================
|
|
32
|
+
"modelName": "Llama-3-1-70b",
|
|
33
|
+
"modelId": "meta.llama3-1-70b-instruct-v1:0",
|
|
34
|
+
"bos_text": "<|begin_of_text|>",
|
|
35
|
+
"role_system_message_prefix": "",
|
|
36
|
+
"role_system_message_suffix": "",
|
|
37
|
+
"role_system_prefix": "<|start_header_id|>",
|
|
38
|
+
"role_system_suffix": "<|end_header_id|>",
|
|
39
|
+
"role_user_message_prefix": "",
|
|
40
|
+
"role_user_message_suffix": "",
|
|
41
|
+
"role_user_prefix": "<|start_header_id|>",
|
|
42
|
+
"role_user_suffix": "<|end_header_id|>",
|
|
43
|
+
"role_assistant_message_prefix": "",
|
|
44
|
+
"role_assistant_message_suffix": "",
|
|
45
|
+
"role_assistant_prefix": "<|start_header_id|>",
|
|
46
|
+
"role_assistant_suffix": "<|end_header_id|>",
|
|
47
|
+
"eom_text": "<|eot_id|>",
|
|
48
|
+
"display_role_names": true,
|
|
49
|
+
"max_tokens_param_name": "max_gen_len",
|
|
50
|
+
"max_supported_response_tokens": 2048,
|
|
51
|
+
"response_chunk_element": "generation",
|
|
52
|
+
},
|
|
3
53
|
{
|
|
4
54
|
// ================
|
|
5
55
|
// == Llama 3 8b ==
|
|
@@ -22,6 +72,7 @@ export const bedrock_models = [
|
|
|
22
72
|
"eom_text": "<|eot_id|>",
|
|
23
73
|
"display_role_names": true,
|
|
24
74
|
"max_tokens_param_name": "max_gen_len",
|
|
75
|
+
"max_supported_response_tokens": 2048,
|
|
25
76
|
"response_chunk_element": "generation",
|
|
26
77
|
},
|
|
27
78
|
{
|
|
@@ -46,6 +97,7 @@ export const bedrock_models = [
|
|
|
46
97
|
"eom_text": "<|eot_id|>",
|
|
47
98
|
"display_role_names": true,
|
|
48
99
|
"max_tokens_param_name": "max_gen_len",
|
|
100
|
+
"max_supported_response_tokens": 2048,
|
|
49
101
|
"response_chunk_element": "generation",
|
|
50
102
|
},
|
|
51
103
|
{
|
|
@@ -70,6 +122,7 @@ export const bedrock_models = [
|
|
|
70
122
|
"eom_text": "</s>",
|
|
71
123
|
"display_role_names": false,
|
|
72
124
|
"max_tokens_param_name": "max_tokens",
|
|
125
|
+
"max_supported_response_tokens": 8192,
|
|
73
126
|
"response_chunk_element": "outputs[0].text",
|
|
74
127
|
},
|
|
75
128
|
{
|
|
@@ -94,6 +147,7 @@ export const bedrock_models = [
|
|
|
94
147
|
"eom_text": "</s>",
|
|
95
148
|
"display_role_names": false,
|
|
96
149
|
"max_tokens_param_name": "max_tokens",
|
|
150
|
+
"max_supported_response_tokens": 4096,
|
|
97
151
|
"response_chunk_element": "outputs[0].text",
|
|
98
152
|
},
|
|
99
153
|
{
|
|
@@ -118,6 +172,7 @@ export const bedrock_models = [
|
|
|
118
172
|
"eom_text": "</s>",
|
|
119
173
|
"display_role_names": false,
|
|
120
174
|
"max_tokens_param_name": "max_tokens",
|
|
175
|
+
"max_supported_response_tokens": 8192,
|
|
121
176
|
"response_chunk_element": "outputs[0].text",
|
|
122
177
|
},
|
|
123
178
|
];
|
package/bedrock-wrapper.js
CHANGED
|
@@ -96,11 +96,13 @@ export async function* bedrockWrapper(awsCreds, openaiChatCompletionsCreateObjec
|
|
|
96
96
|
console.log(`\nPrompt: ${prompt}\n`);
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
+
const max_gen_tokens = max_tokens <= awsModel.max_supported_response_tokens ? max_tokens : awsModel.max_supported_response_tokens;
|
|
100
|
+
|
|
99
101
|
// Format the request payload using the model's native structure.
|
|
100
102
|
const request = {
|
|
101
103
|
prompt,
|
|
102
104
|
// Optional inference parameters:
|
|
103
|
-
[awsModel.max_tokens_param_name]:
|
|
105
|
+
[awsModel.max_tokens_param_name]: max_gen_tokens,
|
|
104
106
|
temperature: temperature,
|
|
105
107
|
top_p: top_p,
|
|
106
108
|
};
|
package/example.js
CHANGED
|
@@ -35,11 +35,11 @@ console.log(`\nsupported models:\n${JSON.stringify(await listBedrockWrapperSuppo
|
|
|
35
35
|
const messages = [
|
|
36
36
|
{
|
|
37
37
|
role: "system",
|
|
38
|
-
content: "You are a helpful AI assistant that follows instructions extremely well. Answer the user questions accurately. Think step by step before answering the question.
|
|
38
|
+
content: "You are a helpful AI assistant that follows instructions extremely well. Answer the user questions accurately. Think step by step before answering the question.",
|
|
39
39
|
},
|
|
40
40
|
{
|
|
41
41
|
role: "user",
|
|
42
|
-
content: "Describe
|
|
42
|
+
content: "Describe what the openai api standard used by lots of serverless LLM api providers is and why it has been widely adopted.",
|
|
43
43
|
},
|
|
44
44
|
{
|
|
45
45
|
role: "assistant",
|
|
@@ -61,7 +61,7 @@ const awsCreds = {
|
|
|
61
61
|
// ----------------------------------------------------------------------
|
|
62
62
|
const openaiChatCompletionsCreateObject = {
|
|
63
63
|
"messages": messages,
|
|
64
|
-
"model": "Llama-3-8b",
|
|
64
|
+
"model": "Llama-3-1-8b",
|
|
65
65
|
"max_tokens": LLM_MAX_GEN_TOKENS,
|
|
66
66
|
"stream": true,
|
|
67
67
|
"temperature": LLM_TEMPERATURE,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bedrock-wrapper",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "🪨 Bedrock Wrapper is an npm package that simplifies the integration of existing OpenAI-compatible API objects with AWS Bedrock's serverless inference LLMs.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"author": "",
|
|
22
22
|
"license": "ISC",
|
|
23
23
|
"dependencies": {
|
|
24
|
-
"@aws-sdk/client-bedrock-runtime": "^3.
|
|
24
|
+
"@aws-sdk/client-bedrock-runtime": "^3.616.0",
|
|
25
25
|
"dotenv": "^16.4.5"
|
|
26
26
|
}
|
|
27
27
|
}
|