bedrock-wrapper 2.1.2 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. package/CHANGELOG.md +20 -0
  2. package/README.md +81 -20
  3. package/bedrock-models.js +507 -392
  4. package/bedrock-wrapper.js +223 -36
  5. package/docs/bedrock-wrapper.jpg +0 -0
  6. package/interactive-example.js +1 -0
  7. package/package.json +42 -35
  8. package/test-image.jpg +0 -0
  9. package/test-models-output.txt +25 -4
  10. package/test-models.js +145 -145
  11. package/test-vision.js +71 -0
  12. package/utils.js +27 -27
package/bedrock-models.js CHANGED
@@ -1,393 +1,508 @@
1
- // Description: This file contains the model configurations
2
-
3
- // NOTE: Not all models are available in all regions.
4
- // Check the bedrock documentation for availability.
5
- // The Llama 3.2 modelId's reference cross-region profile ids.
6
- // https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/cross-region-inference
7
-
8
- export const bedrock_models = [
9
- {
10
- // ==========================
11
- // == Claude 3.5 Sonnet v2 ==
12
- // ==========================
13
- "modelName": "Claude-3-5-Sonnet-v2",
14
- "modelId": "anthropic.claude-3-5-sonnet-20241022-v2:0",
15
- "messages_api": true,
16
- "system_as_separate_field": true,
17
- "display_role_names": true,
18
- "max_tokens_param_name": "max_tokens",
19
- "max_supported_response_tokens": 8192,
20
- "response_chunk_element": "delta.text",
21
- "response_nonchunk_element": "content[0].text",
22
- "special_request_schema": {
23
- "anthropic_version": "bedrock-2023-05-31"
24
- }
25
- },
26
- {
27
- // =======================
28
- // == Claude 3.5 Sonnet ==
29
- // =======================
30
- "modelName": "Claude-3-5-Sonnet",
31
- "modelId": "anthropic.claude-3-5-sonnet-20240620-v1:0",
32
- "messages_api": true,
33
- "system_as_separate_field": true,
34
- "display_role_names": true,
35
- "max_tokens_param_name": "max_tokens",
36
- "max_supported_response_tokens": 8192,
37
- "response_chunk_element": "delta.text",
38
- "response_nonchunk_element": "content[0].text",
39
- "special_request_schema": {
40
- "anthropic_version": "bedrock-2023-05-31"
41
- }
42
- },
43
- {
44
- // ======================
45
- // == Claude 3.5 Haiku ==
46
- // ======================
47
- "modelName": "Claude-3-5-Haiku",
48
- "modelId": "anthropic.claude-3-5-haiku-20241022-v1:0",
49
- "messages_api": true,
50
- "system_as_separate_field": true,
51
- "display_role_names": true,
52
- "max_tokens_param_name": "max_tokens",
53
- "max_supported_response_tokens": 8192,
54
- "response_chunk_element": "delta.text",
55
- "response_nonchunk_element": "content[0].text",
56
- "special_request_schema": {
57
- "anthropic_version": "bedrock-2023-05-31"
58
- }
59
- },
60
- {
61
- // ====================
62
- // == Claude 3 Haiku ==
63
- // ====================
64
- "modelName": "Claude-3-Haiku",
65
- "modelId": "anthropic.claude-3-haiku-20240307-v1:0",
66
- "messages_api": true,
67
- "system_as_separate_field": true,
68
- "display_role_names": true,
69
- "max_tokens_param_name": "max_tokens",
70
- "max_supported_response_tokens": 8192,
71
- "response_chunk_element": "delta.text",
72
- "response_nonchunk_element": "content[0].text",
73
- "special_request_schema": {
74
- "anthropic_version": "bedrock-2023-05-31"
75
- }
76
- },
77
- {
78
- // ==================
79
- // == Llama 3.2 1b ==
80
- // ==================
81
- "modelName": "Llama-3-2-1b",
82
- // "modelId": "meta.llama3-2-1b-instruct-v1:0",
83
- "modelId": "us.meta.llama3-2-1b-instruct-v1:0",
84
- "messages_api": false,
85
- "bos_text": "<|begin_of_text|>",
86
- "role_system_message_prefix": "",
87
- "role_system_message_suffix": "",
88
- "role_system_prefix": "<|start_header_id|>",
89
- "role_system_suffix": "<|end_header_id|>",
90
- "role_user_message_prefix": "",
91
- "role_user_message_suffix": "",
92
- "role_user_prefix": "<|start_header_id|>",
93
- "role_user_suffix": "<|end_header_id|>",
94
- "role_assistant_message_prefix": "",
95
- "role_assistant_message_suffix": "",
96
- "role_assistant_prefix": "<|start_header_id|>",
97
- "role_assistant_suffix": "<|end_header_id|>",
98
- "eom_text": "<|eot_id|>",
99
- "display_role_names": true,
100
- "max_tokens_param_name": "max_gen_len",
101
- "max_supported_response_tokens": 2048,
102
- "response_chunk_element": "generation",
103
- },
104
- {
105
- // ==================
106
- // == Llama 3.2 3b ==
107
- // ==================
108
- "modelName": "Llama-3-2-3b",
109
- // "modelId": "meta.llama3-2-3b-instruct-v1:0",
110
- "modelId": "us.meta.llama3-2-3b-instruct-v1:0",
111
- "messages_api": false,
112
- "bos_text": "<|begin_of_text|>",
113
- "role_system_message_prefix": "",
114
- "role_system_message_suffix": "",
115
- "role_system_prefix": "<|start_header_id|>",
116
- "role_system_suffix": "<|end_header_id|>",
117
- "role_user_message_prefix": "",
118
- "role_user_message_suffix": "",
119
- "role_user_prefix": "<|start_header_id|>",
120
- "role_user_suffix": "<|end_header_id|>",
121
- "role_assistant_message_prefix": "",
122
- "role_assistant_message_suffix": "",
123
- "role_assistant_prefix": "<|start_header_id|>",
124
- "role_assistant_suffix": "<|end_header_id|>",
125
- "eom_text": "<|eot_id|>",
126
- "display_role_names": true,
127
- "max_tokens_param_name": "max_gen_len",
128
- "max_supported_response_tokens": 2048,
129
- "response_chunk_element": "generation",
130
- },
131
- {
132
- // ===================
133
- // == Llama 3.2 11b ==
134
- // ===================
135
- "modelName": "Llama-3-2-11b",
136
- // "modelId": "meta.llama3-2-11b-instruct-v1:0",
137
- "modelId": "us.meta.llama3-2-11b-instruct-v1:0",
138
- "messages_api": false,
139
- "bos_text": "<|begin_of_text|>",
140
- "role_system_message_prefix": "",
141
- "role_system_message_suffix": "",
142
- "role_system_prefix": "<|start_header_id|>",
143
- "role_system_suffix": "<|end_header_id|>",
144
- "role_user_message_prefix": "",
145
- "role_user_message_suffix": "",
146
- "role_user_prefix": "<|start_header_id|>",
147
- "role_user_suffix": "<|end_header_id|>",
148
- "role_assistant_message_prefix": "",
149
- "role_assistant_message_suffix": "",
150
- "role_assistant_prefix": "<|start_header_id|>",
151
- "role_assistant_suffix": "<|end_header_id|>",
152
- "eom_text": "<|eot_id|>",
153
- "display_role_names": true,
154
- "max_tokens_param_name": "max_gen_len",
155
- "max_supported_response_tokens": 2048,
156
- "response_chunk_element": "generation",
157
- },
158
- {
159
- // ===================
160
- // == Llama 3.2 90b ==
161
- // ===================
162
- "modelName": "Llama-3-2-90b",
163
- // "modelId": "meta.llama3-2-90b-instruct-v1:0",
164
- "modelId": "us.meta.llama3-2-90b-instruct-v1:0",
165
- "messages_api": false,
166
- "bos_text": "<|begin_of_text|>",
167
- "role_system_message_prefix": "",
168
- "role_system_message_suffix": "",
169
- "role_system_prefix": "<|start_header_id|>",
170
- "role_system_suffix": "<|end_header_id|>",
171
- "role_user_message_prefix": "",
172
- "role_user_message_suffix": "",
173
- "role_user_prefix": "<|start_header_id|>",
174
- "role_user_suffix": "<|end_header_id|>",
175
- "role_assistant_message_prefix": "",
176
- "role_assistant_message_suffix": "",
177
- "role_assistant_prefix": "<|start_header_id|>",
178
- "role_assistant_suffix": "<|end_header_id|>",
179
- "eom_text": "<|eot_id|>",
180
- "display_role_names": true,
181
- "max_tokens_param_name": "max_gen_len",
182
- "max_supported_response_tokens": 2048,
183
- "response_chunk_element": "generation",
184
- },
185
- {
186
- // ==================
187
- // == Llama 3.1 8b ==
188
- // ==================
189
- "modelName": "Llama-3-1-8b",
190
- "modelId": "meta.llama3-1-8b-instruct-v1:0",
191
- "messages_api": false,
192
- "bos_text": "<|begin_of_text|>",
193
- "role_system_message_prefix": "",
194
- "role_system_message_suffix": "",
195
- "role_system_prefix": "<|start_header_id|>",
196
- "role_system_suffix": "<|end_header_id|>",
197
- "role_user_message_prefix": "",
198
- "role_user_message_suffix": "",
199
- "role_user_prefix": "<|start_header_id|>",
200
- "role_user_suffix": "<|end_header_id|>",
201
- "role_assistant_message_prefix": "",
202
- "role_assistant_message_suffix": "",
203
- "role_assistant_prefix": "<|start_header_id|>",
204
- "role_assistant_suffix": "<|end_header_id|>",
205
- "eom_text": "<|eot_id|>",
206
- "display_role_names": true,
207
- "max_tokens_param_name": "max_gen_len",
208
- "max_supported_response_tokens": 2048,
209
- "response_chunk_element": "generation",
210
- },
211
- {
212
- // ===================
213
- // == Llama 3.1 70b ==
214
- // ===================
215
- "modelName": "Llama-3-1-70b",
216
- "modelId": "meta.llama3-1-70b-instruct-v1:0",
217
- "messages_api": false,
218
- "bos_text": "<|begin_of_text|>",
219
- "role_system_message_prefix": "",
220
- "role_system_message_suffix": "",
221
- "role_system_prefix": "<|start_header_id|>",
222
- "role_system_suffix": "<|end_header_id|>",
223
- "role_user_message_prefix": "",
224
- "role_user_message_suffix": "",
225
- "role_user_prefix": "<|start_header_id|>",
226
- "role_user_suffix": "<|end_header_id|>",
227
- "role_assistant_message_prefix": "",
228
- "role_assistant_message_suffix": "",
229
- "role_assistant_prefix": "<|start_header_id|>",
230
- "role_assistant_suffix": "<|end_header_id|>",
231
- "eom_text": "<|eot_id|>",
232
- "display_role_names": true,
233
- "max_tokens_param_name": "max_gen_len",
234
- "max_supported_response_tokens": 2048,
235
- "response_chunk_element": "generation",
236
- },
237
- {
238
- // ====================
239
- // == Llama 3.1 405b ==
240
- // ====================
241
- "modelName": "Llama-3-1-405b",
242
- "modelId": "meta.llama3-1-405b-instruct-v1:0",
243
- "messages_api": false,
244
- "bos_text": "<|begin_of_text|>",
245
- "role_system_message_prefix": "",
246
- "role_system_message_suffix": "",
247
- "role_system_prefix": "<|start_header_id|>",
248
- "role_system_suffix": "<|end_header_id|>",
249
- "role_user_message_prefix": "",
250
- "role_user_message_suffix": "",
251
- "role_user_prefix": "<|start_header_id|>",
252
- "role_user_suffix": "<|end_header_id|>",
253
- "role_assistant_message_prefix": "",
254
- "role_assistant_message_suffix": "",
255
- "role_assistant_prefix": "<|start_header_id|>",
256
- "role_assistant_suffix": "<|end_header_id|>",
257
- "eom_text": "<|eot_id|>",
258
- "display_role_names": true,
259
- "max_tokens_param_name": "max_gen_len",
260
- "max_supported_response_tokens": 2048,
261
- "response_chunk_element": "generation",
262
- },
263
- {
264
- // ================
265
- // == Llama 3 8b ==
266
- // ================
267
- "modelName": "Llama-3-8b",
268
- "modelId": "meta.llama3-8b-instruct-v1:0",
269
- "messages_api": false,
270
- "bos_text": "<|begin_of_text|>",
271
- "role_system_message_prefix": "",
272
- "role_system_message_suffix": "",
273
- "role_system_prefix": "<|start_header_id|>",
274
- "role_system_suffix": "<|end_header_id|>",
275
- "role_user_message_prefix": "",
276
- "role_user_message_suffix": "",
277
- "role_user_prefix": "<|start_header_id|>",
278
- "role_user_suffix": "<|end_header_id|>",
279
- "role_assistant_message_prefix": "",
280
- "role_assistant_message_suffix": "",
281
- "role_assistant_prefix": "<|start_header_id|>",
282
- "role_assistant_suffix": "<|end_header_id|>",
283
- "eom_text": "<|eot_id|>",
284
- "display_role_names": true,
285
- "max_tokens_param_name": "max_gen_len",
286
- "max_supported_response_tokens": 2048,
287
- "response_chunk_element": "generation",
288
- },
289
- {
290
- // =================
291
- // == Llama 3 70b ==
292
- // =================
293
- "modelName": "Llama-3-70b",
294
- "modelId": "meta.llama3-70b-instruct-v1:0",
295
- "messages_api": false,
296
- "bos_text": "<|begin_of_text|>",
297
- "role_system_message_prefix": "",
298
- "role_system_message_suffix": "",
299
- "role_system_prefix": "<|start_header_id|>",
300
- "role_system_suffix": "<|end_header_id|>",
301
- "role_user_message_prefix": "",
302
- "role_user_message_suffix": "",
303
- "role_user_prefix": "<|start_header_id|>",
304
- "role_user_suffix": "<|end_header_id|>",
305
- "role_assistant_message_prefix": "",
306
- "role_assistant_message_suffix": "",
307
- "role_assistant_prefix": "<|start_header_id|>",
308
- "role_assistant_suffix": "<|end_header_id|>",
309
- "eom_text": "<|eot_id|>",
310
- "display_role_names": true,
311
- "max_tokens_param_name": "max_gen_len",
312
- "max_supported_response_tokens": 2048,
313
- "response_chunk_element": "generation",
314
- },
315
- {
316
- // ================
317
- // == Mistral-7b ==
318
- // ================
319
- "modelName": "Mistral-7b",
320
- "modelId": "mistral.mistral-7b-instruct-v0:2",
321
- "messages_api": false,
322
- "bos_text": "<s>",
323
- "role_system_message_prefix": "",
324
- "role_system_message_suffix": "",
325
- "role_system_prefix": "",
326
- "role_system_suffix": "",
327
- "role_user_message_prefix": "[INST]",
328
- "role_user_message_suffix": "[/INST]",
329
- "role_user_prefix": "",
330
- "role_user_suffix": "",
331
- "role_assistant_message_prefix": "",
332
- "role_assistant_message_suffix": "",
333
- "role_assistant_prefix": "",
334
- "role_assistant_suffix": "",
335
- "eom_text": "</s>",
336
- "display_role_names": false,
337
- "max_tokens_param_name": "max_tokens",
338
- "max_supported_response_tokens": 8192,
339
- "response_chunk_element": "outputs[0].text",
340
- },
341
- {
342
- // ==================
343
- // == Mixtral-8x7b ==
344
- // ==================
345
- "modelName": "Mixtral-8x7b",
346
- "modelId": "mistral.mixtral-8x7b-instruct-v0:1",
347
- "messages_api": false,
348
- "bos_text": "<s>",
349
- "role_system_message_prefix": "",
350
- "role_system_message_suffix": "",
351
- "role_system_prefix": "",
352
- "role_system_suffix": "",
353
- "role_user_message_prefix": "[INST]",
354
- "role_user_message_suffix": "[/INST]",
355
- "role_user_prefix": "",
356
- "role_user_suffix": "",
357
- "role_assistant_message_prefix": "",
358
- "role_assistant_message_suffix": "",
359
- "role_assistant_prefix": "",
360
- "role_assistant_suffix": "",
361
- "eom_text": "</s>",
362
- "display_role_names": false,
363
- "max_tokens_param_name": "max_tokens",
364
- "max_supported_response_tokens": 4096,
365
- "response_chunk_element": "outputs[0].text",
366
- },
367
- {
368
- // ===================
369
- // == Mistral Large ==
370
- // ===================
371
- "modelName": "Mistral-Large",
372
- "modelId": "mistral.mistral-large-2402-v1:0",
373
- "messages_api": false,
374
- "bos_text": "<s>",
375
- "role_system_message_prefix": "",
376
- "role_system_message_suffix": "",
377
- "role_system_prefix": "",
378
- "role_system_suffix": "",
379
- "role_user_message_prefix": "[INST]",
380
- "role_user_message_suffix": "[/INST]",
381
- "role_user_prefix": "",
382
- "role_user_suffix": "",
383
- "role_assistant_message_prefix": "",
384
- "role_assistant_message_suffix": "",
385
- "role_assistant_prefix": "",
386
- "role_assistant_suffix": "",
387
- "eom_text": "</s>",
388
- "display_role_names": false,
389
- "max_tokens_param_name": "max_tokens",
390
- "max_supported_response_tokens": 8192,
391
- "response_chunk_element": "outputs[0].text",
392
- },
1
+ // Description: This file contains the model configurations
2
+
3
+ // NOTE: Not all models are available in all regions.
4
+ // Check the bedrock documentation for availability.
5
+ // The Llama 3.2 modelId's reference cross-region profile ids.
6
+ // https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/cross-region-inference
7
+
8
+ export const bedrock_models = [
9
+ {
10
+ // ================================
11
+ // == Claude 3.7 Sonnet Thinking ==
12
+ // ================================
13
+ "modelName": "Claude-3-7-Sonnet-Thinking",
14
+ // "modelId": "anthropic.claude-3-7-sonnet-20250219-v1:0",
15
+ "modelId": "us.anthropic.claude-3-7-sonnet-20250219-v1:0",
16
+ "vision": true,
17
+ "messages_api": true,
18
+ "system_as_separate_field": true,
19
+ "display_role_names": true,
20
+ "max_tokens_param_name": "max_tokens",
21
+ "max_supported_response_tokens": 131072,
22
+ "response_chunk_element": "delta.text",
23
+ "response_nonchunk_element": "content[0].text",
24
+ "thinking_response_chunk_element": "delta.thinking",
25
+ "thinking_response_nonchunk_element": "content[0].thinking",
26
+ "special_request_schema": {
27
+ "anthropic_version": "bedrock-2023-05-31",
28
+ "anthropic_beta": ["output-128k-2025-02-19"],
29
+ "thinking": {
30
+ "type": "enabled",
31
+ "budget_tokens": 16000
32
+ },
33
+ },
34
+ "image_support": {
35
+ "max_image_size": 20971520, // 20MB
36
+ "supported_formats": ["jpeg", "png", "gif", "webp"],
37
+ "max_images_per_request": 10
38
+ }
39
+ },
40
+ {
41
+ // =======================
42
+ // == Claude 3.7 Sonnet ==
43
+ // =======================
44
+ "modelName": "Claude-3-7-Sonnet",
45
+ // "modelId": "anthropic.claude-3-7-sonnet-20250219-v1:0",
46
+ "modelId": "us.anthropic.claude-3-7-sonnet-20250219-v1:0",
47
+ "vision": true,
48
+ "messages_api": true,
49
+ "system_as_separate_field": true,
50
+ "display_role_names": true,
51
+ "max_tokens_param_name": "max_tokens",
52
+ "max_supported_response_tokens": 131072,
53
+ "response_chunk_element": "delta.text",
54
+ "response_nonchunk_element": "content[0].text",
55
+ "special_request_schema": {
56
+ "anthropic_version": "bedrock-2023-05-31",
57
+ "anthropic_beta": ["output-128k-2025-02-19"]
58
+ },
59
+ "image_support": {
60
+ "max_image_size": 20971520, // 20MB
61
+ "supported_formats": ["jpeg", "png", "gif", "webp"],
62
+ "max_images_per_request": 10
63
+ }
64
+ },
65
+ {
66
+ // ==========================
67
+ // == Claude 3.5 Sonnet v2 ==
68
+ // ==========================
69
+ "modelName": "Claude-3-5-Sonnet-v2",
70
+ "modelId": "anthropic.claude-3-5-sonnet-20241022-v2:0",
71
+ "vision": true,
72
+ "messages_api": true,
73
+ "system_as_separate_field": true,
74
+ "display_role_names": true,
75
+ "max_tokens_param_name": "max_tokens",
76
+ "max_supported_response_tokens": 8192,
77
+ "response_chunk_element": "delta.text",
78
+ "response_nonchunk_element": "content[0].text",
79
+ "special_request_schema": {
80
+ "anthropic_version": "bedrock-2023-05-31"
81
+ },
82
+ "image_support": {
83
+ "max_image_size": 20971520, // 20MB
84
+ "supported_formats": ["jpeg", "png", "gif", "webp"],
85
+ "max_images_per_request": 10
86
+ }
87
+ },
88
+ {
89
+ // =======================
90
+ // == Claude 3.5 Sonnet ==
91
+ // =======================
92
+ "modelName": "Claude-3-5-Sonnet",
93
+ "modelId": "anthropic.claude-3-5-sonnet-20240620-v1:0",
94
+ "vision": true,
95
+ "messages_api": true,
96
+ "system_as_separate_field": true,
97
+ "display_role_names": true,
98
+ "max_tokens_param_name": "max_tokens",
99
+ "max_supported_response_tokens": 8192,
100
+ "response_chunk_element": "delta.text",
101
+ "response_nonchunk_element": "content[0].text",
102
+ "special_request_schema": {
103
+ "anthropic_version": "bedrock-2023-05-31"
104
+ },
105
+ "image_support": {
106
+ "max_image_size": 20971520, // 20MB
107
+ "supported_formats": ["jpeg", "png", "gif", "webp"],
108
+ "max_images_per_request": 10
109
+ }
110
+ },
111
+ {
112
+ // ======================
113
+ // == Claude 3.5 Haiku ==
114
+ // ======================
115
+ "modelName": "Claude-3-5-Haiku",
116
+ "modelId": "anthropic.claude-3-5-haiku-20241022-v1:0",
117
+ "vision": false,
118
+ "messages_api": true,
119
+ "system_as_separate_field": true,
120
+ "display_role_names": true,
121
+ "max_tokens_param_name": "max_tokens",
122
+ "max_supported_response_tokens": 8192,
123
+ "response_chunk_element": "delta.text",
124
+ "response_nonchunk_element": "content[0].text",
125
+ "special_request_schema": {
126
+ "anthropic_version": "bedrock-2023-05-31"
127
+ }
128
+ },
129
+ {
130
+ // ====================
131
+ // == Claude 3 Haiku ==
132
+ // ====================
133
+ "modelName": "Claude-3-Haiku",
134
+ "modelId": "anthropic.claude-3-haiku-20240307-v1:0",
135
+ "vision": false,
136
+ "messages_api": true,
137
+ "system_as_separate_field": true,
138
+ "display_role_names": true,
139
+ "max_tokens_param_name": "max_tokens",
140
+ "max_supported_response_tokens": 8192,
141
+ "response_chunk_element": "delta.text",
142
+ "response_nonchunk_element": "content[0].text",
143
+ "special_request_schema": {
144
+ "anthropic_version": "bedrock-2023-05-31"
145
+ },
146
+ "image_support": {
147
+ "max_image_size": 20971520, // 20MB
148
+ "supported_formats": ["jpeg", "png", "gif", "webp"],
149
+ "max_images_per_request": 10
150
+ }
151
+ },
152
+ {
153
+ // ===================
154
+ // == Llama 3.3 70b ==
155
+ // ===================
156
+ "modelName": "Llama-3-3-70b",
157
+ // "modelId": "meta.llama3-3-70b-instruct-v1:0",
158
+ "modelId": "us.meta.llama3-3-70b-instruct-v1:0",
159
+ "vision": false,
160
+ "messages_api": false,
161
+ "bos_text": "<|begin_of_text|>",
162
+ "role_system_message_prefix": "",
163
+ "role_system_message_suffix": "",
164
+ "role_system_prefix": "<|start_header_id|>",
165
+ "role_system_suffix": "<|end_header_id|>",
166
+ "role_user_message_prefix": "",
167
+ "role_user_message_suffix": "",
168
+ "role_user_prefix": "<|start_header_id|>",
169
+ "role_user_suffix": "<|end_header_id|>",
170
+ "role_assistant_message_prefix": "",
171
+ "role_assistant_message_suffix": "",
172
+ "role_assistant_prefix": "<|start_header_id|>",
173
+ "role_assistant_suffix": "<|end_header_id|>",
174
+ "eom_text": "<|eot_id|>",
175
+ "display_role_names": true,
176
+ "max_tokens_param_name": "max_gen_len",
177
+ "max_supported_response_tokens": 2048,
178
+ "response_chunk_element": "generation"
179
+ },
180
+ {
181
+ // ==================
182
+ // == Llama 3.2 1b ==
183
+ // ==================
184
+ "modelName": "Llama-3-2-1b",
185
+ // "modelId": "meta.llama3-2-1b-instruct-v1:0",
186
+ "modelId": "us.meta.llama3-2-1b-instruct-v1:0",
187
+ "vision": false,
188
+ "messages_api": false,
189
+ "bos_text": "<|begin_of_text|>",
190
+ "role_system_message_prefix": "",
191
+ "role_system_message_suffix": "",
192
+ "role_system_prefix": "<|start_header_id|>",
193
+ "role_system_suffix": "<|end_header_id|>",
194
+ "role_user_message_prefix": "",
195
+ "role_user_message_suffix": "",
196
+ "role_user_prefix": "<|start_header_id|>",
197
+ "role_user_suffix": "<|end_header_id|>",
198
+ "role_assistant_message_prefix": "",
199
+ "role_assistant_message_suffix": "",
200
+ "role_assistant_prefix": "<|start_header_id|>",
201
+ "role_assistant_suffix": "<|end_header_id|>",
202
+ "eom_text": "<|eot_id|>",
203
+ "display_role_names": true,
204
+ "max_tokens_param_name": "max_gen_len",
205
+ "max_supported_response_tokens": 2048,
206
+ "response_chunk_element": "generation"
207
+ },
208
+ {
209
+ // ==================
210
+ // == Llama 3.2 3b ==
211
+ // ==================
212
+ "modelName": "Llama-3-2-3b",
213
+ // "modelId": "meta.llama3-2-3b-instruct-v1:0",
214
+ "modelId": "us.meta.llama3-2-3b-instruct-v1:0",
215
+ "vision": false,
216
+ "messages_api": false,
217
+ "bos_text": "<|begin_of_text|>",
218
+ "role_system_message_prefix": "",
219
+ "role_system_message_suffix": "",
220
+ "role_system_prefix": "<|start_header_id|>",
221
+ "role_system_suffix": "<|end_header_id|>",
222
+ "role_user_message_prefix": "",
223
+ "role_user_message_suffix": "",
224
+ "role_user_prefix": "<|start_header_id|>",
225
+ "role_user_suffix": "<|end_header_id|>",
226
+ "role_assistant_message_prefix": "",
227
+ "role_assistant_message_suffix": "",
228
+ "role_assistant_prefix": "<|start_header_id|>",
229
+ "role_assistant_suffix": "<|end_header_id|>",
230
+ "eom_text": "<|eot_id|>",
231
+ "display_role_names": true,
232
+ "max_tokens_param_name": "max_gen_len",
233
+ "max_supported_response_tokens": 2048,
234
+ "response_chunk_element": "generation"
235
+ },
236
+ {
237
+ // ===================
238
+ // == Llama 3.2 11b ==
239
+ // ===================
240
+ "modelName": "Llama-3-2-11b",
241
+ // "modelId": "meta.llama3-2-11b-instruct-v1:0",
242
+ "modelId": "us.meta.llama3-2-11b-instruct-v1:0",
243
+ "vision": false,
244
+ "messages_api": false,
245
+ "bos_text": "<|begin_of_text|>",
246
+ "role_system_message_prefix": "",
247
+ "role_system_message_suffix": "",
248
+ "role_system_prefix": "<|start_header_id|>",
249
+ "role_system_suffix": "<|end_header_id|>",
250
+ "role_user_message_prefix": "",
251
+ "role_user_message_suffix": "",
252
+ "role_user_prefix": "<|start_header_id|>",
253
+ "role_user_suffix": "<|end_header_id|>",
254
+ "role_assistant_message_prefix": "",
255
+ "role_assistant_message_suffix": "",
256
+ "role_assistant_prefix": "<|start_header_id|>",
257
+ "role_assistant_suffix": "<|end_header_id|>",
258
+ "eom_text": "<|eot_id|>",
259
+ "display_role_names": true,
260
+ "max_tokens_param_name": "max_gen_len",
261
+ "max_supported_response_tokens": 2048,
262
+ "response_chunk_element": "generation"
263
+ },
264
+ {
265
+ // ===================
266
+ // == Llama 3.2 90b ==
267
+ // ===================
268
+ "modelName": "Llama-3-2-90b",
269
+ // "modelId": "meta.llama3-2-90b-instruct-v1:0",
270
+ "modelId": "us.meta.llama3-2-90b-instruct-v1:0",
271
+ "vision": false,
272
+ "messages_api": false,
273
+ "bos_text": "<|begin_of_text|>",
274
+ "role_system_message_prefix": "",
275
+ "role_system_message_suffix": "",
276
+ "role_system_prefix": "<|start_header_id|>",
277
+ "role_system_suffix": "<|end_header_id|>",
278
+ "role_user_message_prefix": "",
279
+ "role_user_message_suffix": "",
280
+ "role_user_prefix": "<|start_header_id|>",
281
+ "role_user_suffix": "<|end_header_id|>",
282
+ "role_assistant_message_prefix": "",
283
+ "role_assistant_message_suffix": "",
284
+ "role_assistant_prefix": "<|start_header_id|>",
285
+ "role_assistant_suffix": "<|end_header_id|>",
286
+ "eom_text": "<|eot_id|>",
287
+ "display_role_names": true,
288
+ "max_tokens_param_name": "max_gen_len",
289
+ "max_supported_response_tokens": 2048,
290
+ "response_chunk_element": "generation"
291
+ },
292
+ {
293
+ // ==================
294
+ // == Llama 3.1 8b ==
295
+ // ==================
296
+ "modelName": "Llama-3-1-8b",
297
+ "modelId": "meta.llama3-1-8b-instruct-v1:0",
298
+ "vision": false,
299
+ "messages_api": false,
300
+ "bos_text": "<|begin_of_text|>",
301
+ "role_system_message_prefix": "",
302
+ "role_system_message_suffix": "",
303
+ "role_system_prefix": "<|start_header_id|>",
304
+ "role_system_suffix": "<|end_header_id|>",
305
+ "role_user_message_prefix": "",
306
+ "role_user_message_suffix": "",
307
+ "role_user_prefix": "<|start_header_id|>",
308
+ "role_user_suffix": "<|end_header_id|>",
309
+ "role_assistant_message_prefix": "",
310
+ "role_assistant_message_suffix": "",
311
+ "role_assistant_prefix": "<|start_header_id|>",
312
+ "role_assistant_suffix": "<|end_header_id|>",
313
+ "eom_text": "<|eot_id|>",
314
+ "display_role_names": true,
315
+ "max_tokens_param_name": "max_gen_len",
316
+ "max_supported_response_tokens": 2048,
317
+ "response_chunk_element": "generation"
318
+ },
319
+ {
320
+ // ===================
321
+ // == Llama 3.1 70b ==
322
+ // ===================
323
+ "modelName": "Llama-3-1-70b",
324
+ "modelId": "meta.llama3-1-70b-instruct-v1:0",
325
+ "vision": false,
326
+ "messages_api": false,
327
+ "bos_text": "<|begin_of_text|>",
328
+ "role_system_message_prefix": "",
329
+ "role_system_message_suffix": "",
330
+ "role_system_prefix": "<|start_header_id|>",
331
+ "role_system_suffix": "<|end_header_id|>",
332
+ "role_user_message_prefix": "",
333
+ "role_user_message_suffix": "",
334
+ "role_user_prefix": "<|start_header_id|>",
335
+ "role_user_suffix": "<|end_header_id|>",
336
+ "role_assistant_message_prefix": "",
337
+ "role_assistant_message_suffix": "",
338
+ "role_assistant_prefix": "<|start_header_id|>",
339
+ "role_assistant_suffix": "<|end_header_id|>",
340
+ "eom_text": "<|eot_id|>",
341
+ "display_role_names": true,
342
+ "max_tokens_param_name": "max_gen_len",
343
+ "max_supported_response_tokens": 2048,
344
+ "response_chunk_element": "generation"
345
+ },
346
+ {
347
+ // ====================
348
+ // == Llama 3.1 405b ==
349
+ // ====================
350
+ "modelName": "Llama-3-1-405b",
351
+ "modelId": "meta.llama3-1-405b-instruct-v1:0",
352
+ "vision": false,
353
+ "messages_api": false,
354
+ "bos_text": "<|begin_of_text|>",
355
+ "role_system_message_prefix": "",
356
+ "role_system_message_suffix": "",
357
+ "role_system_prefix": "<|start_header_id|>",
358
+ "role_system_suffix": "<|end_header_id|>",
359
+ "role_user_message_prefix": "",
360
+ "role_user_message_suffix": "",
361
+ "role_user_prefix": "<|start_header_id|>",
362
+ "role_user_suffix": "<|end_header_id|>",
363
+ "role_assistant_message_prefix": "",
364
+ "role_assistant_message_suffix": "",
365
+ "role_assistant_prefix": "<|start_header_id|>",
366
+ "role_assistant_suffix": "<|end_header_id|>",
367
+ "eom_text": "<|eot_id|>",
368
+ "display_role_names": true,
369
+ "max_tokens_param_name": "max_gen_len",
370
+ "max_supported_response_tokens": 2048,
371
+ "response_chunk_element": "generation"
372
+ },
373
+ {
374
+ // ================
375
+ // == Llama 3 8b ==
376
+ // ================
377
+ "modelName": "Llama-3-8b",
378
+ "modelId": "meta.llama3-8b-instruct-v1:0",
379
+ "vision": false,
380
+ "messages_api": false,
381
+ "bos_text": "<|begin_of_text|>",
382
+ "role_system_message_prefix": "",
383
+ "role_system_message_suffix": "",
384
+ "role_system_prefix": "<|start_header_id|>",
385
+ "role_system_suffix": "<|end_header_id|>",
386
+ "role_user_message_prefix": "",
387
+ "role_user_message_suffix": "",
388
+ "role_user_prefix": "<|start_header_id|>",
389
+ "role_user_suffix": "<|end_header_id|>",
390
+ "role_assistant_message_prefix": "",
391
+ "role_assistant_message_suffix": "",
392
+ "role_assistant_prefix": "<|start_header_id|>",
393
+ "role_assistant_suffix": "<|end_header_id|>",
394
+ "eom_text": "<|eot_id|>",
395
+ "display_role_names": true,
396
+ "max_tokens_param_name": "max_gen_len",
397
+ "max_supported_response_tokens": 2048,
398
+ "response_chunk_element": "generation"
399
+ },
400
+ {
401
+ // =================
402
+ // == Llama 3 70b ==
403
+ // =================
404
+ "modelName": "Llama-3-70b",
405
+ "modelId": "meta.llama3-70b-instruct-v1:0",
406
+ "vision": false,
407
+ "messages_api": false,
408
+ "bos_text": "<|begin_of_text|>",
409
+ "role_system_message_prefix": "",
410
+ "role_system_message_suffix": "",
411
+ "role_system_prefix": "<|start_header_id|>",
412
+ "role_system_suffix": "<|end_header_id|>",
413
+ "role_user_message_prefix": "",
414
+ "role_user_message_suffix": "",
415
+ "role_user_prefix": "<|start_header_id|>",
416
+ "role_user_suffix": "<|end_header_id|>",
417
+ "role_assistant_message_prefix": "",
418
+ "role_assistant_message_suffix": "",
419
+ "role_assistant_prefix": "<|start_header_id|>",
420
+ "role_assistant_suffix": "<|end_header_id|>",
421
+ "eom_text": "<|eot_id|>",
422
+ "display_role_names": true,
423
+ "max_tokens_param_name": "max_gen_len",
424
+ "max_supported_response_tokens": 2048,
425
+ "response_chunk_element": "generation"
426
+ },
427
+ {
428
+ // ================
429
+ // == Mistral-7b ==
430
+ // ================
431
+ "modelName": "Mistral-7b",
432
+ "modelId": "mistral.mistral-7b-instruct-v0:2",
433
+ "vision": false,
434
+ "messages_api": false,
435
+ "bos_text": "<s>",
436
+ "role_system_message_prefix": "",
437
+ "role_system_message_suffix": "",
438
+ "role_system_prefix": "",
439
+ "role_system_suffix": "",
440
+ "role_user_message_prefix": "[INST]",
441
+ "role_user_message_suffix": "[/INST]",
442
+ "role_user_prefix": "",
443
+ "role_user_suffix": "",
444
+ "role_assistant_message_prefix": "",
445
+ "role_assistant_message_suffix": "",
446
+ "role_assistant_prefix": "",
447
+ "role_assistant_suffix": "",
448
+ "eom_text": "</s>",
449
+ "display_role_names": false,
450
+ "max_tokens_param_name": "max_tokens",
451
+ "max_supported_response_tokens": 8192,
452
+ "response_chunk_element": "outputs[0].text"
453
+ },
454
+ {
455
+ // ==================
456
+ // == Mixtral-8x7b ==
457
+ // ==================
458
+ "modelName": "Mixtral-8x7b",
459
+ "modelId": "mistral.mixtral-8x7b-instruct-v0:1",
460
+ "vision": false,
461
+ "messages_api": false,
462
+ "bos_text": "<s>",
463
+ "role_system_message_prefix": "",
464
+ "role_system_message_suffix": "",
465
+ "role_system_prefix": "",
466
+ "role_system_suffix": "",
467
+ "role_user_message_prefix": "[INST]",
468
+ "role_user_message_suffix": "[/INST]",
469
+ "role_user_prefix": "",
470
+ "role_user_suffix": "",
471
+ "role_assistant_message_prefix": "",
472
+ "role_assistant_message_suffix": "",
473
+ "role_assistant_prefix": "",
474
+ "role_assistant_suffix": "",
475
+ "eom_text": "</s>",
476
+ "display_role_names": false,
477
+ "max_tokens_param_name": "max_tokens",
478
+ "max_supported_response_tokens": 4096,
479
+ "response_chunk_element": "outputs[0].text"
480
+ },
481
+ {
482
+ // ===================
483
+ // == Mistral Large ==
484
+ // ===================
485
+ "modelName": "Mistral-Large",
486
+ "modelId": "mistral.mistral-large-2402-v1:0",
487
+ "vision": false,
488
+ "messages_api": false,
489
+ "bos_text": "<s>",
490
+ "role_system_message_prefix": "",
491
+ "role_system_message_suffix": "",
492
+ "role_system_prefix": "",
493
+ "role_system_suffix": "",
494
+ "role_user_message_prefix": "[INST]",
495
+ "role_user_message_suffix": "[/INST]",
496
+ "role_user_prefix": "",
497
+ "role_user_suffix": "",
498
+ "role_assistant_message_prefix": "",
499
+ "role_assistant_message_suffix": "",
500
+ "role_assistant_prefix": "",
501
+ "role_assistant_suffix": "",
502
+ "eom_text": "</s>",
503
+ "display_role_names": false,
504
+ "max_tokens_param_name": "max_tokens",
505
+ "max_supported_response_tokens": 8192,
506
+ "response_chunk_element": "outputs[0].text"
507
+ },
393
508
  ];