@aws/ml-container-creator 0.2.6 → 0.3.0

package/servers/lib/catalogs/model-servers.json

@@ -64,7 +64,157 @@
                     "notes": "Requires instance with 4+ GPUs. Set TENSOR_PARALLEL_SIZE to match GPU count"
                 }
             },
-            "notes": "vLLM 0.4.0 adds prefix caching and improved performance. Requires CUDA 12.0+"
+            "notes": "vLLM 0.4.0 adds prefix caching and improved performance. Requires CUDA 12.0+",
+            "supportedModelTypes": [
+                "arcee",
+                "arctic",
+                "aria",
+                "aya_vision",
+                "baichuan",
+                "bailing_moe",
+                "bamba",
+                "bart",
+                "bert",
+                "bert_with_rope",
+                "blip2",
+                "bloom",
+                "chameleon",
+                "chatglm",
+                "cohere2_vision",
+                "commandr",
+                "dbrx",
+                "deepseek",
+                "deepseek_mtp",
+                "deepseek_v2",
+                "deepseek_vl2",
+                "dots1",
+                "ernie45",
+                "ernie45_moe",
+                "exaone",
+                "exaone4",
+                "fairseq2_llama",
+                "falcon",
+                "falcon_h1",
+                "florence2",
+                "fuyu",
+                "gemma",
+                "gemma2",
+                "gemma3",
+                "gemma3_mm",
+                "gemma3n",
+                "gemma3n_mm",
+                "glm",
+                "glm4",
+                "glm4_1v",
+                "glm4_moe",
+                "glm4_moe_mtp",
+                "glm4v",
+                "gpt2",
+                "gpt_bigcode",
+                "gpt_j",
+                "gpt_neox",
+                "gpt_oss",
+                "granite",
+                "granite_speech",
+                "granitemoe",
+                "granitemoehybrid",
+                "granitemoeshared",
+                "gritlm",
+                "grok1",
+                "h2ovl",
+                "hunyuan_v1",
+                "hyperclovax_vision",
+                "idefics3",
+                "internlm2",
+                "internlm2_ve",
+                "interns1",
+                "internvl",
+                "jais",
+                "jamba",
+                "jina_vl",
+                "keye",
+                "kimi_vl",
+                "llama",
+                "llama4",
+                "llama4_eagle",
+                "llama_eagle",
+                "llama_eagle3",
+                "llava",
+                "llava_next",
+                "llava_next_video",
+                "llava_onevision",
+                "mamba",
+                "mamba2",
+                "medusa",
+                "mimo",
+                "mimo_mtp",
+                "minicpm",
+                "minicpm3",
+                "minicpm_eagle",
+                "minicpmo",
+                "minicpmv",
+                "minimax_text_01",
+                "minimax_vl_01",
+                "mistral3",
+                "mixtral",
+                "mixtral_quant",
+                "mllama",
+                "mllama4",
+                "mlp_speculator",
+                "modernbert",
+                "molmo",
+                "mpt",
+                "nemotron",
+                "nemotron_h",
+                "nemotron_nas",
+                "nemotron_vl",
+                "nvlm_d",
+                "olmo",
+                "olmo2",
+                "olmoe",
+                "opt",
+                "orion",
+                "ovis",
+                "paligemma",
+                "persimmon",
+                "phi",
+                "phi3",
+                "phi3v",
+                "phi4_multimodal",
+                "phi4flash",
+                "phi4mm",
+                "phimoe",
+                "pixtral",
+                "plamo2",
+                "prithvi_geospatial_mae",
+                "qwen",
+                "qwen2",
+                "qwen2_5_omni_thinker",
+                "qwen2_5_vl",
+                "qwen2_audio",
+                "qwen2_moe",
+                "qwen2_rm",
+                "qwen2_vl",
+                "qwen3",
+                "qwen3_moe",
+                "qwen_vl",
+                "roberta",
+                "skyworkr1v",
+                "smolvlm",
+                "solar",
+                "stablelm",
+                "starcoder2",
+                "step3_text",
+                "step3_vl",
+                "tarsier",
+                "telechat2",
+                "teleflm",
+                "transformers",
+                "ultravox",
+                "voxtral",
+                "whisper",
+                "zamba2"
+            ]
         },
         {
             "image": "vllm/vllm-openai:v0.9.1",
@@ -130,7 +280,133 @@
                     "notes": "Requires instance with 4+ GPUs. Set TENSOR_PARALLEL_SIZE to match GPU count"
                 }
             },
-            "notes": "vLLM 0.4.0 adds prefix caching and improved performance. Requires CUDA 12.0+"
+            "notes": "vLLM 0.4.0 adds prefix caching and improved performance. Requires CUDA 12.0+",
+            "supportedModelTypes": [
+                "arctic",
+                "aria",
+                "aya_vision",
+                "baichuan",
+                "bamba",
+                "bart",
+                "bert",
+                "bert_with_rope",
+                "blip2",
+                "bloom",
+                "chameleon",
+                "chatglm",
+                "commandr",
+                "dbrx",
+                "deepseek",
+                "deepseek_mtp",
+                "deepseek_v2",
+                "deepseek_vl2",
+                "eagle",
+                "exaone",
+                "fairseq2_llama",
+                "falcon",
+                "falcon_h1",
+                "florence2",
+                "fuyu",
+                "gemma",
+                "gemma2",
+                "gemma3",
+                "gemma3_mm",
+                "glm",
+                "glm4",
+                "glm4v",
+                "gpt2",
+                "gpt_bigcode",
+                "gpt_j",
+                "gpt_neox",
+                "granite",
+                "granite_speech",
+                "granitemoe",
+                "granitemoehybrid",
+                "granitemoeshared",
+                "gritlm",
+                "grok1",
+                "h2ovl",
+                "idefics3",
+                "internlm2",
+                "internlm2_ve",
+                "internvl",
+                "jais",
+                "jamba",
+                "kimi_vl",
+                "llama",
+                "llama_eagle",
+                "llama_eagle3",
+                "llava",
+                "llava_next",
+                "llava_next_video",
+                "llava_onevision",
+                "mamba",
+                "mamba2",
+                "medusa",
+                "mimo",
+                "mimo_mtp",
+                "minicpm",
+                "minicpm3",
+                "minicpm_eagle",
+                "minicpmo",
+                "minicpmv",
+                "minimax_text_01",
+                "minimax_vl_01",
+                "mistral3",
+                "mixtral",
+                "mixtral_quant",
+                "mllama",
+                "mllama4",
+                "mlp_speculator",
+                "modernbert",
+                "molmo",
+                "mpt",
+                "nemotron",
+                "nemotron_h",
+                "nemotron_nas",
+                "nvlm_d",
+                "olmo",
+                "olmo2",
+                "olmoe",
+                "opt",
+                "orion",
+                "ovis",
+                "paligemma",
+                "persimmon",
+                "phi",
+                "phi3",
+                "phi3_small",
+                "phi3v",
+                "phi4mm",
+                "phimoe",
+                "pixtral",
+                "plamo2",
+                "prithvi_geospatial_mae",
+                "qwen",
+                "qwen2",
+                "qwen2_5_omni_thinker",
+                "qwen2_5_vl",
+                "qwen2_audio",
+                "qwen2_moe",
+                "qwen2_rm",
+                "qwen2_vl",
+                "qwen3",
+                "qwen3_moe",
+                "qwen_vl",
+                "roberta",
+                "skyworkr1v",
+                "smolvlm",
+                "solar",
+                "stablelm",
+                "starcoder2",
+                "tarsier",
+                "telechat2",
+                "teleflm",
+                "transformers",
+                "ultravox",
+                "whisper",
+                "zamba2"
+            ]
         }
     ],
     "sglang": [
@@ -311,7 +587,8 @@
                     "notes": "Enables running larger models on smaller instances with acceptable accuracy"
                 }
             },
-            "notes": "TensorRT-LLM 1.0.0 adds chunked context and INT4 support. Requires CUDA 12.1+"
+            "notes": "TensorRT-LLM 1.0.0 adds chunked context and INT4 support. Requires CUDA 12.1+",
+            "supportedModelTypes": []
         },
         {
             "image": "nvcr.io/nvidia/tensorrt-llm/release:1.1.0",
@@ -380,7 +657,8 @@
                     "notes": "Enables running larger models on smaller instances with acceptable accuracy"
                 }
             },
-            "notes": "TensorRT-LLM 1.0.0 adds chunked context and INT4 support. Requires CUDA 12.1+"
+            "notes": "TensorRT-LLM 1.0.0 adds chunked context and INT4 support. Requires CUDA 12.1+",
+            "supportedModelTypes": []
         }
     ],
     "lmi": [
@@ -958,4 +1236,4 @@
             "notes": "Triton Python backend for custom model serving with TritonPythonModel interface. GPU optional"
         }
     ]
-}
+}

package/servers/lib/catalogs/models.json

@@ -1,6 +1,9 @@
 {
     "openai/gpt-oss-20b": {
         "family": "gpt-oss",
+        "parameterCount": 20000000000,
+        "defaultDtype": "bfloat16",
+        "maxPositionEmbeddings": 8192,
         "gated": false,
         "tags": [
             "text-generation",
@@ -99,6 +102,9 @@
     },
     "meta-llama/Llama-2-70b-chat-hf": {
         "family": "llama-2",
+        "parameterCount": 70000000000,
+        "defaultDtype": "float16",
+        "maxPositionEmbeddings": 4096,
         "gated": true,
         "tags": [
             "text-generation",
@@ -259,6 +265,30 @@
             "text-generation"
         ]
     },
+    "meta-llama/Llama-2-70b-hf": {
+        "family": "llama-2",
+        "parameterCount": 70000000000,
+        "defaultDtype": "float16",
+        "maxPositionEmbeddings": 4096,
+        "gated": true,
+        "tags": [
+            "text-generation",
+            "llama-2"
+        ],
+        "architecture": "LlamaForCausalLM",
+        "notes": "Llama-2 70B base model (non-chat). Requires multi-GPU for inference.",
+        "chatTemplate": "",
+        "frameworkCompatibility": {
+            "vllm": ">=0.3.0",
+            "tensorrt-llm": ">=0.8.0",
+            "sglang": ">=0.2.0"
+        },
+        "validationLevel": "community-validated",
+        "modelType": "transformer",
+        "tasks": [
+            "text-generation"
+        ]
+    },
     "meta-llama/Llama-2-*": {
         "family": "llama-2",
         "gated": true,

package/servers/lib/schemas/image-catalog.schema.json

@@ -145,6 +145,12 @@
                 },
                 "notes": {
                     "type": "string"
+                },
+                "supportedModelTypes": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
                 }
             },
             "additionalProperties": false

package/servers/model-picker/index.js

@@ -195,11 +195,12 @@ class HuggingFaceResolver extends ModelResolver {
         }
 
         // Fetch model config (conditional)
-        if (!fields || fields.includes('architecture')) {
+        if (!fields || fields.includes('architecture') || fields.includes('model_type')) {
             const modelConfig = await this._fetchJson(
                 `${this.baseUrl}/${modelId}/resolve/main/config.json`
             )
             metadata.architecture = modelConfig?.architectures?.[0] || null
+            metadata.model_type = modelConfig?.model_type || null
         }
 
         return Object.keys(metadata).length > 0 ? metadata : null

package/src/app.js

@@ -119,6 +119,23 @@ export async function run(projectName, options) {
     let answers;
     if (configManager.shouldSkipPrompts()) {
         console.log('\n🚀 Skipping prompts - using configuration from other sources');
+
+        // Fail-fast if required parameters are missing
+        const missing = configManager.getMissingRequiredParameters();
+        if (missing.length > 0) {
+            console.error('\n❌ Cannot skip prompts — required parameters are missing:\n');
+            for (const param of missing) {
+                const matrix = configManager._getParameterMatrix()[param];
+                const cliFlag = matrix?.cliOption ? `--${matrix.cliOption}` : '';
+                const envVar = matrix?.envVar || '';
+                const hints = [cliFlag, envVar].filter(Boolean).join(' or ');
+                console.error(`   • ${param}${hints ? ` (${hints})` : ''}`);
+            }
+            console.error('\n   Provide these via CLI flags, environment variables, or a config file.');
+            console.error('   Run "ml-container-creator --help" for available options.\n');
+            process.exit(1);
+        }
+
         answers = configManager.getFinalConfiguration();
 
         // Infer modelSource from model name prefix if not set
@@ -493,7 +510,9 @@ async function _ensureTemplateVariables(answers, registryConfigManager = null) {
         chatTemplate: null,
         chatTemplateSource: null,
         hfToken: null,
+        hfTokenArn: null,
         ngcApiKey: null,
+        ngcTokenArn: null,
         envVars: {},
         inferenceAmiVersion: null,
         accelerator: null,

package/src/lib/architecture-sync.js

@@ -0,0 +1,171 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Architecture Sync
+ *
+ * Fetches model registry source files from server GitHub repositories
+ * and extracts supported model_type values into the model-servers catalog.
+ */
+
+import { readFileSync, writeFileSync } from 'node:fs';
+
+/**
+ * Parse vLLM's model registry Python source to extract model_type keys.
+ *
+ * vLLM's registry maps architecture class names to (module, impl_class) tuples:
+ *   "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
+ *   "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
+ *
+ * The module name (first tuple element) corresponds to the model_type.
+ * Also matches older formats where model_type is used directly as dict key.
+ *
+ * @param {string} source - Python source code content
+ * @returns {string[]} Sorted array of model_type strings
+ */
+export const parseVllmRegistry = (source) => {
+    const modelTypes = new Set();
+    const patterns = [
+        // Tuple value format: ("module_name", "ClassName") — extract module_name
+        /\("([a-z][a-z0-9_]*)"\s*,\s*"[A-Z]/g,
+        // Direct lowercase key format (older registries): "model_type": (
+        /"([a-z][a-z0-9_]*)":\s*\(/g,
+        // Direct lowercase key format: "model_type": ClassName
+        /"([a-z][a-z0-9_]*)":\s*[A-Z]/g,
+        // Direct lowercase key format: "model_type": [
+        /"([a-z][a-z0-9_]*)":\s*\[/g
+    ];
+    for (const pattern of patterns) {
+        let match;
+        while ((match = pattern.exec(source)) !== null) {
+            modelTypes.add(match[1]);
+        }
+    }
+    return [...modelTypes].sort();
+};
+
+/**
+ * Parse SGLang's model_registry.py to extract model_type keys.
+ *
+ * Matches patterns like:
+ *   "llama": ModelClass,
+ *   "qwen2": (ModulePath, ClassName),
+ *
+ * @param {string} source - Python source code content
+ * @returns {string[]} Sorted array of model_type strings
+ */
+export const parseSglangRegistry = (source) => {
+    const modelTypes = new Set();
+    const patterns = [
+        /"([a-z][a-z0-9_]*)":\s*\(/g,
+        /"([a-z][a-z0-9_]*)":\s*[A-Z]/g,
+        /"([a-z][a-z0-9_]*)":\s*\[/g
+    ];
+    for (const pattern of patterns) {
+        let match;
+        while ((match = pattern.exec(source)) !== null) {
+            modelTypes.add(match[1]);
+        }
+    }
+    return [...modelTypes].sort();
+};
+
+/**
+ * Parse TensorRT-LLM's models __init__.py to extract model_type keys.
+ *
+ * Matches patterns from MODEL_MAP or similar dict structures:
+ *   "llama": LlamaForCausalLM,
+ *   "gpt2": GPT2LMHeadModel,
+ *
+ * @param {string} source - Python source code content
+ * @returns {string[]} Sorted array of model_type strings
+ */
+export const parseTensorRTRegistry = (source) => {
+    const modelTypes = new Set();
+    const patterns = [
+        /"([a-z][a-z0-9_]*)":\s*[A-Z]/g,
+        /"([a-z][a-z0-9_]*)":\s*\(/g,
+        /'([a-z][a-z0-9_]*)':\s*[A-Z]/g,
+        /'([a-z][a-z0-9_]*)':\s*\(/g
+    ];
+    for (const pattern of patterns) {
+        let match;
+        while ((match = pattern.exec(source)) !== null) {
+            modelTypes.add(match[1]);
+        }
+    }
+    return [...modelTypes].sort();
+};
+
+/**
+ * Configuration mapping each server to its GitHub repository,
+ * registry file path, tag prefix, and parser function.
+ */
+export const SERVER_REGISTRY_SOURCES = {
+    vllm: {
+        repo: 'vllm-project/vllm',
+        file: 'vllm/model_executor/models/registry.py',
+        tagPrefix: 'v',
+        parser: parseVllmRegistry
+    },
+    sglang: {
+        repo: 'sgl-project/sglang',
+        file: 'python/sglang/srt/models/model_registry.py',
+        tagPrefix: 'v',
+        parser: parseSglangRegistry
+    },
+    'tensorrt-llm': {
+        repo: 'NVIDIA/TensorRT-LLM',
+        file: 'tensorrt_llm/models/__init__.py',
+        tagPrefix: 'v',
+        parser: parseTensorRTRegistry
+    }
+};
+
+/**
+ * Sync supported model architectures from server GitHub repositories
+ * into the model-servers catalog.
+ *
+ * For each server entry in the catalog that has a matching source config,
+ * fetches the model registry file from GitHub at the version tag and
+ * parses it to extract supported model_type values.
+ *
+ * @param {string} catalogPath - Path to model-servers.json
+ * @returns {object} Summary with counts and failures
+ */
+export const syncArchitectures = async (catalogPath) => {
+    const catalog = JSON.parse(readFileSync(catalogPath, 'utf8'));
+    const summary = { servers: [], failures: [] };
+
+    for (const [server, entries] of Object.entries(catalog)) {
+        const source = SERVER_REGISTRY_SOURCES[server];
+        if (!source) continue;
+
+        for (const entry of entries) {
+            const version = entry.labels?.framework_version;
+            if (!version) continue;
+
+            const tag = `${source.tagPrefix}${version}`;
+            const url = `https://raw.githubusercontent.com/${source.repo}/${tag}/${source.file}`;
+
+            try {
+                const response = await fetch(url);
+                if (!response.ok) {
+                    summary.failures.push({ server, version, reason: `HTTP ${response.status}` });
+                    console.log(`   ⚠️  ${server} ${version}: fetch failed (HTTP ${response.status})`);
+                    continue;
+                }
+                const content = await response.text();
+                entry.supportedModelTypes = source.parser(content);
+                summary.servers.push({ server, version, count: entry.supportedModelTypes.length });
+                console.log(`   ✓ ${server} ${version}: ${entry.supportedModelTypes.length} architectures`);
+            } catch (err) {
+                summary.failures.push({ server, version, reason: err.message });
+                console.log(`   ⚠️  ${server} ${version}: fetch failed (${err.message})`);
+            }
+        }
+    }
+
+    writeFileSync(catalogPath, JSON.stringify(catalog, null, 4));
+    return summary;
+};

package/src/lib/arn-detection.js

@@ -0,0 +1,22 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * ARN Detection Utility
+ *
+ * Provides a pure function for distinguishing AWS Secrets Manager ARNs
+ * from plaintext values. Used by the prompt flow and CLI to determine
+ * whether user input should be treated as a secret reference or a
+ * literal token value.
+ */
+
+const SECRETS_MANAGER_ARN_PREFIX = 'arn:aws:secretsmanager:';
+
+/**
+ * Determines if a value is a Secrets Manager ARN.
+ * @param {*} value - The input value to check
+ * @returns {boolean} True if the value is a Secrets Manager ARN
+ */
+export function isSecretsManagerArn(value) {
+    return typeof value === 'string' && value.startsWith(SECRETS_MANAGER_ARN_PREFIX);
+}