@aws/ml-container-creator 0.2.0

package/src/prompt-adapter.js

@@ -0,0 +1,63 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { select, input, confirm, checkbox, number, Separator } from '@inquirer/prompts'
+
+/**
+ * Maps Yeoman prompt type names to @inquirer/prompts runner functions.
+ */
+const runners = { list: select, select, input, confirm, checkbox, number }
+
+/**
+ * Runs a sequence of Yeoman-style prompt definitions using @inquirer/prompts.
+ *
+ * Handles:
+ * - Type mapping (list → select)
+ * - Conditional prompts via `when` function
+ * - Dynamic `choices`, `default`, and `message` (functions resolved with current answers)
+ * - Separator mapping from Yeoman format to @inquirer/prompts Separator
+ * - Validate function passthrough
+ *
+ * @param {Array<object>} prompts - Array of Yeoman-style prompt definitions
+ * @param {object} [previousAnswers={}] - Answers from prior prompt phases
+ * @param {object} [options={}] - Options for dependency injection
+ * @param {object} [options.runners] - Override prompt runners (useful for testing)
+ * @returns {Promise<object>} Accumulated answers keyed by prompt name
+ */
+export async function runPrompts(prompts, previousAnswers = {}, options = {}) {
+    const promptRunners = options.runners || runners
+    const answers = { ...previousAnswers }
+
+    for (const prompt of prompts) {
+        if (prompt.when && !prompt.when(answers)) continue
+
+        const type = prompt.type === 'list' ? 'select' : prompt.type
+        const runner = promptRunners[type]
+
+        if (!runner) {
+            throw new Error(`Unsupported prompt type: "${prompt.type}"`)
+        }
+
+        const message = typeof prompt.message === 'function'
+            ? prompt.message(answers) : prompt.message
+        const choices = typeof prompt.choices === 'function'
+            ? prompt.choices(answers) : prompt.choices
+        const defaultVal = typeof prompt.default === 'function'
+            ? prompt.default(answers) : prompt.default
+
+        const mappedChoices = choices?.map(c =>
+            c && c.type === 'separator'
+                ? new Separator(c.separator || c.line)
+                : c
+        )
+
+        const config = { message }
+        if (mappedChoices !== undefined) config.choices = mappedChoices
+        if (defaultVal !== undefined) config.default = defaultVal
+        if (prompt.validate) config.validate = prompt.validate
+
+        answers[prompt.name] = await runner(config)
+    }
+
+    return answers
+}

package/templates/Dockerfile

@@ -0,0 +1,300 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+<% if (comments && comments.acceleratorInfo) { %>
+<%= comments.acceleratorInfo %>
+<% } %>
+
+<% if (comments && comments.validationInfo) { %>
+<%= comments.validationInfo %>
+<% } %>
+
+<% if (framework !== 'transformers') { %>
+FROM <%= baseImage || 'python:3.12-slim' %>
+
+# Set a docker label to name this project, postpended with the build time
+LABEL project.name="<%= projectName %>-<%= buildTimestamp %>" \
+      project.base-name="<%= projectName %>" \
+      project.build-time="<%= buildTimestamp %>"
+
+# Set a docker label to advertise multi-model support on the container
+LABEL com.amazonaws.sagemaker.capabilities.multi-models=true
+# Set a docker label to enable container to use SAGEMAKER_BIND_TO_PORT environment variable if present
+LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true
+
+# Set working directory
+WORKDIR /opt/ml
+
+RUN apt-get update &&  \
+    apt-get upgrade -y && \
+    apt-get clean
+
+# Install system dependencies
+RUN apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    curl \
+    git \
+    nginx \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# Install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir pip==24.2 && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Copy model serving code
+COPY code/serve.py code/
+COPY code/start_server.py code/
+COPY code/model_handler.py code/
+
+<% if (modelServer === 'flask') { %>
+COPY code/flask/gunicorn_config.py code/
+COPY code/flask/wsgi.py code/
+<% } %>
+
+
+# Set up SageMaker directories
+RUN mkdir -p /opt/ml/input/data \
+    && mkdir -p /opt/ml/output/data \
+    && mkdir -p /opt/ml/model
+
+COPY nginx-predictors.conf /etc/nginx/nginx.conf
+
+# Model files will be provided at runtime via SageMaker model artifacts
+<% if (includeSampleModel) { %>
+# Copy the generated sample model
+<% if (modelFormat === 'SavedModel') { %>
+COPY sample_model/abalone_model /opt/ml/model/
+<% } else { %>
+COPY sample_model/abalone_model.<%= modelFormat %> /opt/ml/model/
+<% } %>
+# Also copy training script for reference
+COPY sample_model/ /opt/ml/sample_model/
+<% } else { %>
+# COPY your_model_files /opt/ml/model/
+<% } %>
+
+<% if (comments && comments.envVarExplanations && Object.keys(comments.envVarExplanations).length > 0) { %>
+# Environment Variables Configuration
+<% for (const [category, comment] of Object.entries(comments.envVarExplanations)) { %>
+<%= comment %>
+<% } %>
+<% } %>
+
+# Set environment variables for SageMaker
+ENV PYTHONPATH=/opt/ml/code
+ENV SAGEMAKER_BIND_TO_PORT=8080
+
+<% if (orderedEnvVars && orderedEnvVars.length > 0) { %>
+# Additional environment variables from configuration
+<% orderedEnvVars.forEach(({ key, value }) => { %>
+ENV <%= key %>=<%= value %>
+<% }); %>
+<% } %>
+
+# Expose port 8080 for SageMaker inference
+EXPOSE 8080
+
+<% if (comments && comments.troubleshooting) { %>
+<%= comments.troubleshooting %>
+<% } %>
+
+# Set the inference script as the entry point
+RUN chmod +x code/start_server.py
+ENTRYPOINT ["python", "/opt/ml/code/start_server.py"]
+<% } else { %>
+<% if (comments && comments.acceleratorInfo) { %>
+<%= comments.acceleratorInfo %>
+<% } %>
+
+<% if (comments && comments.validationInfo) { %>
+<%= comments.validationInfo %>
+<% } %>
+
+<% if (modelServer === 'vllm') { %>
+# https://github.com/aws-samples/sagemaker-genai-hosting-examples/tree/main/OpenAI/gpt-oss/deploy/docker
+ARG BASE_IMAGE=<%= baseImage || 'vllm/vllm-openai:v0.10.1' %>
+<% } else if (modelServer === 'sglang') { %>
+ARG BASE_IMAGE=<%= baseImage || 'lmsysorg/sglang:v0.5.4.post1' %>
+<% } else if (modelServer === 'tensorrt-llm') { %>
+# TensorRT-LLM requires NVIDIA NGC authentication
+# Before building, authenticate with NGC:
+#   1. Create NGC account: https://ngc.nvidia.com/signup
+#   2. Generate API key: https://ngc.nvidia.com/setup/api-key
+#   3. Login: docker login nvcr.io
+#      Username: $oauthtoken
+#      Password: <your-ngc-api-key>
+# Using a stable release for better SageMaker compatibility
+ARG BASE_IMAGE=<%= baseImage || 'nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc8' %>
+<% } else if (modelServer === 'lmi') { %>
+# AWS Large Model Inference (LMI) Container
+# LMI containers are pre-built by AWS and include DJL Serving with optimized inference libraries
+# Available backends: vLLM, TensorRT-LLM, LMI-Dist (DeepSpeed), Transformers NeuronX
+# Documentation: https://docs.djl.ai/master/docs/serving/serving/docs/lmi/index.html
+ARG BASE_IMAGE=<%= baseImage || '763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.32.0-lmi14.0.0-cu126' %>
+<% } else if (modelServer === 'djl') { %>
+# DJL Serving Container
+# Deep Java Library serving with support for multiple inference backends
+# Documentation: https://djl.ai/
+ARG BASE_IMAGE=<%= baseImage || 'deepjavalibrary/djl-serving:0.36.0-pytorch-gpu' %>
+<% } %>
+
+FROM ${BASE_IMAGE}
+
+<% if (comments && comments.chatTemplate) { %>
+<%= comments.chatTemplate %>
+<% } %>
+
+# Model source metadata
+ENV MODEL_SOURCE="<%= (typeof modelSource !== 'undefined' && modelSource) ? modelSource : 'huggingface' %>"
+<% if (typeof artifactUri !== 'undefined' && artifactUri) { %>
+ENV MODEL_ARTIFACT_URI="<%= artifactUri %>"
+<% } %>
+
+# Set the model name for the transformer model
+<% if (modelServer === 'vllm') { %>
+<% if (typeof modelLoadStrategy !== 'undefined' && modelLoadStrategy === 'build-time' && (typeof modelSource === 'undefined' || !modelSource || modelSource === 'huggingface')) { %>
+ENV VLLM_MODEL="/opt/ml/model"
+<% } else { %>
+ENV VLLM_MODEL="<%= modelName %>"
+<% } %>
+<% if (typeof modelSource !== 'undefined' && modelSource && modelSource !== 'huggingface') { %>
+# Model will be resolved at container startup by the serve script
+<% } %>
+<% } else if (modelServer === 'sglang') { %>
+<% if (typeof modelLoadStrategy !== 'undefined' && modelLoadStrategy === 'build-time' && (typeof modelSource === 'undefined' || !modelSource || modelSource === 'huggingface')) { %>
+ENV SGLANG_MODEL_PATH="/opt/ml/model"
+<% } else { %>
+ENV SGLANG_MODEL_PATH="<%= modelName %>"
+<% } %>
+<% if (typeof modelSource !== 'undefined' && modelSource && modelSource !== 'huggingface') { %>
+# Model will be resolved at container startup by the serve script
+<% } %>
+<% } else if (modelServer === 'tensorrt-llm') { %>
+<% if (typeof modelLoadStrategy !== 'undefined' && modelLoadStrategy === 'build-time' && (typeof modelSource === 'undefined' || !modelSource || modelSource === 'huggingface')) { %>
+ENV TRTLLM_MODEL="/opt/ml/model"
+<% } else { %>
+ENV TRTLLM_MODEL="<%= modelName %>"
+<% } %>
+<% if (typeof modelSource !== 'undefined' && modelSource && modelSource !== 'huggingface') { %>
+# Model will be resolved at container startup by the serve script
+<% } %>
+
+# Disable UCX CUDA transport to avoid symbol lookup errors
+# The UCX CUDA library has compatibility issues with some CUDA versions in SageMaker
+RUN if [ -f /usr/local/ucx/lib/ucx/libuct_cuda.so.0 ]; then \
+        mv /usr/local/ucx/lib/ucx/libuct_cuda.so.0 /usr/local/ucx/lib/ucx/libuct_cuda.so.0.disabled; \
+    fi
+
+ENV UCX_TLS=tcp,self,sm
+ENV UCX_NET_DEVICES=all
+ENV NCCL_IB_DISABLE=1
+ENV NCCL_P2P_DISABLE=1
+<% } else if (modelServer === 'lmi' || modelServer === 'djl') { %>
+# LMI/DJL Configuration
+# Model configuration is done via serving.properties file
+# The model will be loaded from HuggingFace Hub or S3
+ENV HF_MODEL_ID="<%= modelName %>"
+<% if (typeof modelSource !== 'undefined' && modelSource && modelSource !== 'huggingface') { %>
+# Model will be resolved at container startup by the serve script
+<% } %>
+
+# DJL Serving listens on port 8080 by default (SageMaker requirement)
+ENV SERVING_PORT=8080
+<% } %>
+
+<% if (hfToken && (!modelSource || modelSource === 'huggingface')) { %>
+# Set HuggingFace authentication token
+ENV HF_TOKEN="<%= hfToken %>"
+<% } %>
+
+<% if (chatTemplate) { %>
+# Chat template configuration
+# This template formats chat messages for the model
+# Writing to file to avoid shell escaping issues with multi-line Jinja2 templates
+COPY code/chat_template.jinja /opt/ml/chat_template.jinja
+ENV SGLANG_CHAT_TEMPLATE="/opt/ml/chat_template.jinja"
+<% } %>
+
+<% if (comments && comments.envVarExplanations && Object.keys(comments.envVarExplanations).length > 0) { %>
+# Environment Variables Configuration
+<% for (const [category, comment] of Object.entries(comments.envVarExplanations)) { %>
+<%= comment %>
+<% } %>
+<% } %>
+
+<% if (orderedEnvVars && orderedEnvVars.length > 0) { %>
+# Additional environment variables from configuration
+<% orderedEnvVars.forEach(({ key, value }) => { %>
+ENV <%= key %>=<%= value %>
+<% }); %>
+<% } %>
+
+<% if (typeof modelSource !== 'undefined' && modelSource && modelSource !== 'huggingface' && modelServer !== 'lmi' && modelServer !== 'djl') { %>
+# Install AWS CLI for S3 model downloads
+RUN pip install --no-cache-dir awscli
+
+<% } %>
+<% if (typeof modelLoadStrategy !== 'undefined' && modelLoadStrategy === 'build-time') { %>
+# Build-time model download
+# ⚠️  Credentials required during docker build.
+#   Build with: DOCKER_BUILDKIT=1 docker build --secret id=aws,src=$HOME/.aws/credentials .
+#   Or in CodeBuild: pass AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN as build args.
+<% if (typeof modelSource === 'undefined' || !modelSource || modelSource === 'huggingface') { %>
+ARG HF_TOKEN
+RUN huggingface-cli download <%= modelName %> --local-dir /opt/ml/model
+<% } else if (typeof artifactUri !== 'undefined' && artifactUri) { %>
+ARG AWS_ACCESS_KEY_ID
+ARG AWS_SECRET_ACCESS_KEY
+ARG AWS_SESSION_TOKEN
+ARG AWS_DEFAULT_REGION=<%= (typeof awsRegion !== 'undefined' && awsRegion) ? awsRegion : 'us-east-1' %>
+RUN AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
+    AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
+    AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} \
+    AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION} \
+    aws s3 sync <%= artifactUri %> /opt/ml/model
+<% } %>
+<% } %>
+<% if (modelServer === 'tensorrt-llm') { %>
+# Install nginx and curl for reverse proxy and health checks
+RUN apt-get update && \
+    apt-get install -y nginx curl && \
+    rm -rf /var/lib/apt/lists/*
+
+# Copy nginx configuration for TensorRT-LLM
+COPY nginx-tensorrt.conf /etc/nginx/nginx.conf
+
+# Copy TensorRT-LLM serve script
+COPY code/serve /usr/bin/serve_trtllm
+RUN chmod +x /usr/bin/serve_trtllm
+
+# Copy startup script
+COPY code/start_server.sh /usr/bin/start_server.sh
+RUN chmod +x /usr/bin/start_server.sh
+
+ENTRYPOINT [ "/usr/bin/start_server.sh" ]
+<% } else if (modelServer === 'lmi' || modelServer === 'djl') { %>
+# Create serving.properties configuration file for LMI/DJL
+RUN mkdir -p /opt/ml/model
+COPY code/serving.properties /opt/ml/model/serving.properties
+
+<% if (comments && comments.troubleshooting) { %>
+<%= comments.troubleshooting %>
+<% } %>
+
+# LMI/DJL containers use their own entrypoint
+# The container will automatically start DJL Serving with the configuration
+<% } else { %>
+COPY code/serve /usr/bin/serve
+RUN chmod 777 /usr/bin/serve
+
+<% if (comments && comments.troubleshooting) { %>
+<%= comments.troubleshooting %>
+<% } %>
+
+ENTRYPOINT [ "/usr/bin/serve" ]
+<% } %>
+
+<% } %>

package/templates/IAM_PERMISSIONS.md

@@ -0,0 +1,84 @@
+# IAM Permissions — <%= projectName %>
+
+## Overview
+
+This project uses three sets of IAM permissions:
+
+1. **SageMaker Execution Role** — created automatically by `bootstrap` via CloudFormation
+2. **CodeBuild Service Role** — created automatically by `./do/submit`
+3. **User/CI Permissions** — your AWS user or CI system needs these to run the do-scripts
+
+## SageMaker Execution Role
+
+The bootstrap command creates an IAM role (`mlcc-sagemaker-execution-role`) with permissions for:
+
+- **SageMaker**: Create, update, delete, and invoke endpoints, endpoint configs, models, and inference components
+- **ECR**: Pull images from the `ml-container-creator` repository
+- **CloudWatch Logs**: Write container logs
+- **S3**: Read model artifacts from `ml-container-creator-*` buckets
+
+The role is defined in the CloudFormation stack template (`config/bootstrap-stack.json`) and updated automatically when you re-run bootstrap after upgrading.
+
+If you use a custom role (`--role-arn`), ensure it has at minimum:
+
+| Permission | Purpose |
+|-----------|---------|
+| `sagemaker:CreateEndpoint`, `CreateEndpointConfig`, `CreateModel`, `CreateInferenceComponent` | Deploy |
+| `sagemaker:DeleteEndpoint`, `DeleteEndpointConfig`, `DeleteModel`, `DeleteInferenceComponent` | Clean up |
+| `sagemaker:DescribeEndpoint`, `DescribeEndpointConfig`, `DescribeModel`, `DescribeInferenceComponent` | Status checks |
+| `sagemaker:InvokeEndpoint`, `InvokeEndpointAsync` | Inference |
+| `sagemaker:UpdateEndpoint`, `UpdateEndpointWeightsAndCapacities`, `UpdateInferenceComponent` | Updates |
+| `ecr:GetAuthorizationToken`, `BatchGetImage`, `GetDownloadUrlForLayer`, `BatchCheckLayerAvailability` | Pull container image |
+| `logs:CreateLogGroup`, `CreateLogStream`, `PutLogEvents` | Container logging |
+| `s3:GetObject`, `s3:ListBucket` on `ml-container-creator-*` | Model artifact access |
+
+Trust policy must allow `sagemaker.amazonaws.com` to assume the role.
+
+## CodeBuild Service Role
+
+Created automatically by `./do/submit` as `<%= codebuildProjectName %>-service-role`. Permissions:
+
+- **CloudWatch Logs**: Write build logs to `/aws/codebuild/<%= codebuildProjectName %>*`
+- **ECR**: Push images to `ml-container-creator` repository
+- **S3**: Read source archives from `codebuild-source-*` buckets
+
+## User/CI Permissions
+
+Your AWS user or CI system needs these permissions to run the do-scripts:
+
+| Script | Permissions Needed |
+|--------|-------------------|
+| `./do/push` | `ecr:GetAuthorizationToken`, `ecr:PutImage`, `ecr:InitiateLayerUpload`, `ecr:UploadLayerPart`, `ecr:CompleteLayerUpload`, `ecr:BatchCheckLayerAvailability` |
+| `./do/submit` | `codebuild:CreateProject`, `codebuild:StartBuild`, `codebuild:BatchGetBuilds`, `iam:CreateRole`, `iam:PutRolePolicy`, `iam:PassRole`, `s3:PutObject`, `s3:CreateBucket` |
+| `./do/deploy` | `sagemaker:CreateEndpointConfig`, `sagemaker:CreateEndpoint`, `sagemaker:CreateInferenceComponent`, `sagemaker:DescribeEndpoint`, `iam:PassRole` |
+| `./do/clean` | `sagemaker:DeleteEndpoint`, `sagemaker:DeleteEndpointConfig`, `sagemaker:DeleteInferenceComponent`, `codebuild:DeleteProject`, `iam:DeleteRole`, `iam:DeleteRolePolicy` |
+| `./do/test` | `sagemaker-runtime:InvokeEndpoint` |
+| `bootstrap` | `cloudformation:*`, `iam:CreateRole`, `iam:PutRolePolicy`, `iam:TagRole`, `ecr:CreateRepository`, `s3:CreateBucket` (and `sts:GetCallerIdentity`) |
+
+<% if (framework === 'transformers' && hfToken) { %>
+## HuggingFace Token Security
+
+This project includes a HuggingFace token baked into the Docker image. Key practices:
+
+- **Use read-only tokens** — never bake write tokens into containers
+- **Rotate regularly** — every 30–90 days, or immediately if compromised
+- **Restrict ECR access** — limit who can pull images containing the token
+- **Consider runtime injection** — pass `HF_TOKEN` as a SageMaker environment variable instead of baking it in (avoids token in image layers, enables rotation without rebuild)
+
+To rotate: generate a new token on [HuggingFace](https://huggingface.co/settings/tokens), rebuild with `./do/submit`, revoke the old token.
+
+If compromised: revoke the token immediately, delete the ECR image (`aws ecr batch-delete-image`), rebuild, and review CloudTrail logs.
+<% } %>
+
+## Security Best Practices
+
+- **Least privilege**: All roles are scoped to specific resources where possible
+- **Resource scoping**: CodeBuild permissions scoped to `<%= codebuildProjectName %>`, SageMaker to `<%= projectName %>*`
+- **Audit**: Enable CloudTrail for IAM, SageMaker, ECR, and CodeBuild events
+- **Separate environments**: Consider per-environment roles (dev/prod)
+
+## References
+
+- [SageMaker Execution Roles](https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-roles.html)
+- [CodeBuild Service Role](https://docs.aws.amazon.com/codebuild/latest/userguide/setting-up.html#setting-up-service-role)
+- [ECR Permissions](https://docs.aws.amazon.com/AmazonECR/latest/userguide/security_iam_service-with-iam.html)