@aws/ml-container-creator 0.2.0

package/templates/hyperpod/configmap.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: <%= projectName %>-config
+  namespace: <%= hyperPodNamespace %>
+  labels:
+    app: <%= projectName %>
+    managed-by: ml-container-creator
+data:
+  DEPLOYMENT_CONFIG: "<%= deploymentConfig %>"
+  FRAMEWORK: "<%= framework %>"
+  MODEL_SERVER: "<%= modelServer %>"
+<% if (framework === 'transformers') { %>
+  MODEL_NAME: "<%= modelName %>"
+<% if (hfToken) { %>
+  HF_TOKEN: "<%= hfToken %>"
+<% } %>
+<% if (ngcApiKey) { %>
+  NGC_API_KEY: "<%= ngcApiKey %>"
+<% } %>
+<% } %>
+<% if (modelFormat) { %>
+  MODEL_FORMAT: "<%= modelFormat %>"
+<% } %>

package/templates/hyperpod/deployment.yaml

@@ -0,0 +1,71 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: <%= projectName %>
+  namespace: <%= hyperPodNamespace %>
+  labels:
+    app: <%= projectName %>
+    framework: <%= framework %>
+    managed-by: ml-container-creator
+spec:
+  replicas: <%= hyperPodReplicas %>
+  selector:
+    matchLabels:
+      app: <%= projectName %>
+  template:
+    metadata:
+      labels:
+        app: <%= projectName %>
+        framework: <%= framework %>
+    spec:
+      containers:
+        - name: <%= projectName %>
+          image: ${AWS_ACCOUNT_ID}.dkr.ecr.<%= awsRegion %>.amazonaws.com/ml-container-creator:<%= projectName %>-latest
+          ports:
+            - containerPort: 8080
+              protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: <%= projectName %>-config
+          resources:
+            requests:
+              cpu: "4"
+              memory: "16Gi"
+              nvidia.com/gpu: "1"
+            limits:
+              cpu: "8"
+              memory: "32Gi"
+              nvidia.com/gpu: "1"
+<% if (fsxVolumeHandle) { %>
+          volumeMounts:
+            - name: fsx-storage
+              mountPath: /opt/ml/model
+<% } %>
+          readinessProbe:
+            httpGet:
+              path: /ping
+              port: 8080
+            initialDelaySeconds: 120
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: 8080
+            initialDelaySeconds: 900
+            periodSeconds: 30
+            failureThreshold: 5
+      nodeSelector:
+        node.kubernetes.io/instance-type: <%= instanceType %>
+      tolerations:
+        - key: "nvidia.com/gpu"
+          operator: "Exists"
+          effect: "NoSchedule"
+        - key: "sagemaker.amazonaws.com/hyperpod"
+          operator: "Exists"
+          effect: "NoSchedule"
+<% if (fsxVolumeHandle) { %>
+      volumes:
+        - name: fsx-storage
+          persistentVolumeClaim:
+            claimName: <%= projectName %>-fsx-pvc
+<% } %>

package/templates/hyperpod/pvc.yaml

@@ -0,0 +1,42 @@
+<% if (fsxVolumeHandle) { %>
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: <%= projectName %>-fsx-pvc
+  namespace: <%= hyperPodNamespace %>
+  labels:
+    app: <%= projectName %>
+    managed-by: ml-container-creator
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: fsx-lustre
+  resources:
+    requests:
+      storage: 1200Gi
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: <%= projectName %>-fsx-pv
+  labels:
+    app: <%= projectName %>
+    managed-by: ml-container-creator
+spec:
+  capacity:
+    storage: 1200Gi
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: fsx-lustre
+  csi:
+    driver: fsx.csi.aws.com
+    volumeHandle: <%= fsxVolumeHandle %>
+    volumeAttributes:
+      dnsname: <%= fsxVolumeHandle %>.fsx.<%= awsRegion %>.amazonaws.com
+      mountname: fsx
+<% } else { %>
+# PVC not generated - no FSx volume handle provided
+# To enable FSx for Lustre storage, provide fsxVolumeHandle during generation
+<% } %>

package/templates/hyperpod/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: <%= projectName %>
+  namespace: <%= hyperPodNamespace %>
+  labels:
+    app: <%= projectName %>
+    managed-by: ml-container-creator
+spec:
+  type: ClusterIP
+  ports:
+    - port: 8080
+      targetPort: 8080
+      protocol: TCP
+      name: http
+  selector:
+    app: <%= projectName %>

package/templates/nginx-diffusors.conf

@@ -0,0 +1,74 @@
+#!/bin/bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+worker_processes auto;
+daemon off;
+pid /tmp/nginx.pid;
+error_log /dev/stderr info;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    access_log /dev/stdout;
+    client_max_body_size 100M;
+
+    upstream vllm_omni {
+        server 127.0.0.1:8081;
+    }
+
+    server {
+        listen 8080;
+        server_name _;
+
+        # SageMaker health check endpoint
+        location /ping {
+            proxy_pass http://vllm_omni/health;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_connect_timeout 2s;
+            proxy_read_timeout 2s;
+        }
+
+        # SageMaker inference endpoint - map to OpenAI image generation
+        location /invocations {
+            proxy_pass http://vllm_omni/v1/images/generations;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_connect_timeout 300s;
+            proxy_read_timeout 300s;
+            proxy_send_timeout 300s;
+        }
+
+        # Pass through all OpenAI API endpoints for direct access
+        location /v1/ {
+            proxy_pass http://vllm_omni;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_connect_timeout 300s;
+            proxy_read_timeout 300s;
+            proxy_send_timeout 300s;
+        }
+
+        # Health endpoint passthrough
+        location /health {
+            proxy_pass http://vllm_omni/health;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        # Metrics endpoint passthrough
+        location /metrics {
+            proxy_pass http://vllm_omni/metrics;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+    }
+}

package/templates/nginx-predictors.conf

@@ -0,0 +1,47 @@
+#!/bin/bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+worker_processes auto;
+daemon off;
+pid /tmp/nginx.pid;
+error_log /dev/stderr info;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    access_log /dev/stdout;
+    client_max_body_size 100M;
+
+    upstream backend {
+        server 127.0.0.1:8000;
+    }
+
+    server {
+        listen 8080;
+        server_name _;
+
+        # SageMaker health check endpoint
+        location /ping {
+            proxy_pass http://backend/ping;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_connect_timeout 2s;
+            proxy_read_timeout 2s;
+        }
+
+        # SageMaker inference endpoint
+        location /invocations {
+            proxy_pass http://backend/invocations;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_connect_timeout 300s;
+            proxy_read_timeout 300s;
+            proxy_send_timeout 300s;
+        }
+    }
+}

package/templates/nginx-tensorrt.conf

@@ -0,0 +1,74 @@
+#!/bin/bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+worker_processes auto;
+daemon off;
+pid /tmp/nginx.pid;
+error_log /dev/stderr info;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    access_log /dev/stdout;
+    client_max_body_size 100M;
+
+    upstream trtllm {
+        server 127.0.0.1:8081;
+    }
+
+    server {
+        listen 8080;
+        server_name _;
+
+        # SageMaker health check endpoint
+        location /ping {
+            proxy_pass http://trtllm/health;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_connect_timeout 2s;
+            proxy_read_timeout 2s;
+        }
+
+        # SageMaker inference endpoint - map to OpenAI chat completions
+        location /invocations {
+            proxy_pass http://trtllm/v1/chat/completions;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_connect_timeout 300s;
+            proxy_read_timeout 300s;
+            proxy_send_timeout 300s;
+        }
+
+        # Pass through all OpenAI API endpoints for direct access
+        location /v1/ {
+            proxy_pass http://trtllm;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_connect_timeout 300s;
+            proxy_read_timeout 300s;
+            proxy_send_timeout 300s;
+        }
+
+        # Health endpoint passthrough
+        location /health {
+            proxy_pass http://trtllm/health;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        # Metrics endpoint passthrough
+        location /metrics {
+            proxy_pass http://trtllm/metrics;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+    }
+}

package/templates/requirements.txt

@@ -0,0 +1,61 @@
+<% if (framework === 'sklearn') { %># Scikit-learn dependencies
+scikit-learn==1.7.1
+joblib==1.4.2
+<% } else if (framework === 'xgboost') { %># XGBoost dependencies
+xgboost==2.1.3
+<% } else if (framework === 'tensorflow') { %># Tensorflow dependencies
+setuptools>=65.0.0
+tensorflow==2.20.0
+<% } else if (framework === 'transformers' && modelServer === 'sglang') { %># SGLang dependencies
+sglang[all]==0.5.10
+torch>=2.1.0
+transformers>=4.36.0
+accelerate>=0.25.0
+httpx==0.27.0  # For async HTTP client if needed
+fastapi==0.116.1
+uvicorn==0.35.0
+<% } %>
+numpy==1.26.4
+
+# Web framework for inference endpoint
+<% if (modelServer === 'flask') { %>
+flask==3.0.3
+gunicorn==23.0.0
+<% } else if (modelServer === 'fastapi') { %>
+fastapi==0.116.1
+uvicorn==0.35.0
+<% } %>
+
+# SageMaker dependencies
+sagemaker-inference==1.10.1
+sagemaker-training==5.1.0
+
+# Optional utilities
+psutil==6.0.0  # For system monitoring
+prometheus-client==0.20.0  # For metrics (if needed)
+
+<% if (includeSampleModel && architecture !== 'triton') { %>
+# Sample model dependencies
+ucimlrepo
+pandas
+<% } %>
+
+<% if (includeSampleModel && architecture === 'triton') { %>
+# Triton sample model training dependencies
+ucimlrepo
+pandas
+numpy
+<% if (backend === 'fil' && (modelFormat === 'xgboost_json' || modelFormat === 'xgboost_ubj')) { %>
+xgboost
+<% } else if (backend === 'fil' && modelFormat === 'lightgbm_txt') { %>
+lightgbm
+<% } else if (backend === 'onnxruntime') { %>
+scikit-learn
+skl2onnx
+onnxruntime
+<% } else if (backend === 'tensorflow') { %>
+tensorflow
+<% } else if (backend === 'python') { %>
+scikit-learn
+<% } %>
+<% } %>

package/templates/sample_model/test_inference.py

@@ -0,0 +1,123 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import numpy as np
+<% if (architecture === 'triton') { %>
+<% if (backend === 'fil' && (modelFormat === 'xgboost_json' || modelFormat === 'xgboost_ubj')) { %>
+import xgboost as xgb
+<% } else if (backend === 'fil' && modelFormat === 'lightgbm_txt') { %>
+import lightgbm as lgb
+<% } else if (backend === 'onnxruntime') { %>
+import onnxruntime as ort
+<% } else if (backend === 'tensorflow') { %>
+import tensorflow as tf
+<% } else if (backend === 'python' && modelFormat === 'pkl') { %>
+import pickle
+<% } else if (backend === 'python' && modelFormat === 'joblib') { %>
+import joblib
+<% } %>
+<% } else { %>
+<% if (framework === 'sklearn') { %>
+<% if (modelFormat === 'joblib') { %>import joblib
+<% } else if (modelFormat === 'pkl') { %>import pickle
+<% } %>
+<% } else if (framework === 'xgboost') { %>import xgboost as xgb
+<% } else if (framework === 'tensorflow') { %>import tensorflow as tf
+<% } %>
+<% } %>
+
+# Load the trained model
+<% if (architecture === 'triton') { %>
+<% if (backend === 'fil' && modelFormat === 'xgboost_json') { %>
+model = xgb.Booster()
+model.load_model('./abalone_model.json')
+<% } else if (backend === 'fil' && modelFormat === 'xgboost_ubj') { %>
+model = xgb.Booster()
+model.load_model('./abalone_model.ubj')
+<% } else if (backend === 'fil' && modelFormat === 'lightgbm_txt') { %>
+model = lgb.Booster(model_file='./abalone_model.txt')
+<% } else if (backend === 'onnxruntime') { %>
+session = ort.InferenceSession('./abalone_model.onnx')
+<% } else if (backend === 'tensorflow') { %>
+model = tf.saved_model.load('./abalone_model.savedmodel')
+<% } else if (backend === 'python' && modelFormat === 'pkl') { %>
+with open('./abalone_model.pkl', 'rb') as f:
+    model = pickle.load(f)
+<% } else if (backend === 'python' && modelFormat === 'joblib') { %>
+model = joblib.load('./abalone_model.joblib')
+<% } %>
+<% } else { %>
+<% if (framework === 'sklearn') { %>
+<% if (modelFormat === 'joblib') { %>model = joblib.load('./abalone_model.joblib')
+<% } else if (modelFormat === 'pkl') { -%>
+with open('./abalone_model.pkl', 'rb') as f:
+    model = pickle.load(f)
+<% } %>
+<% } else if (framework === 'xgboost') { %>model = xgb.Booster()
+<% if (modelFormat === 'json') { %>model.load_model('./abalone_model.json')
+<% } else if (modelFormat === 'model') { %>model.load_model('./abalone_model.model')
+<% } else if (modelFormat === 'ubj') { %>model.load_model('./abalone_model.ubj')
+<% } %>
+<% } else if (framework === 'tensorflow') { %>
+<% if (modelFormat === 'keras') { %>model = tf.keras.models.load_model('./abalone_model.keras')
+<% } else if (modelFormat === 'h5') { %>model = tf.keras.models.load_model('./abalone_model.h5')
+<% } else if (modelFormat === 'SavedModel') { %>model = tf.saved_model.load('./abalone_model')
+model = model.signatures['serving_default']
+<% } %>
+<% } %>
+<% } %>
+
+# Create synthetic input array for abalone prediction
+# Features: [Sex, Length, Diameter, Height, Whole_weight, Shucked_weight, Viscera_weight, Shell_weight]
+# Sex: 0=M, 1=F, 2=I (Infant)
+synthetic_input = np.array([[
+    1,      # Sex: Female
+    0.455,  # Length
+    0.365,  # Diameter
+    0.095,  # Height
+    0.514,  # Whole weight
+    0.2245, # Shucked weight
+    0.101,  # Viscera weight
+    0.15    # Shell weight
+]])
+
+# Make prediction
+<% if (architecture === 'triton') { %>
+<% if (backend === 'fil' && (modelFormat === 'xgboost_json' || modelFormat === 'xgboost_ubj')) { %>
+feature_names = ['Sex', 'Length', 'Diameter', 'Height', 'Whole_weight', 'Shucked_weight', 'Viscera_weight', 'Shell_weight']
+dtest = xgb.DMatrix(synthetic_input, feature_names=feature_names)
+prediction = model.predict(dtest)
+print(f"Predicted rings: {prediction[0]:.1f}")
+<% } else if (backend === 'fil' && modelFormat === 'lightgbm_txt') { %>
+prediction = model.predict(synthetic_input)
+print(f"Predicted rings: {prediction[0]:.1f}")
+<% } else if (backend === 'onnxruntime') { %>
+input_name = session.get_inputs()[0].name
+prediction = session.run(None, {input_name: synthetic_input.astype(np.float32)})
+print(f"Predicted rings: {prediction[0][0]:.1f}")
+<% } else if (backend === 'tensorflow') { %>
+infer = model.signatures['serving_default']
+input_tensor = tf.constant(synthetic_input, dtype=tf.float32)
+prediction = infer(input_tensor)
+output_key = list(prediction.keys())[0]
+result = prediction[output_key].numpy()
+print(f"Predicted rings: {result[0][0]:.1f}")
+<% } else if (backend === 'python') { %>
+prediction = model.predict(synthetic_input)
+print(f"Predicted rings: {prediction[0]:.1f}")
+<% } %>
+<% } else { %>
+<% if (framework === 'sklearn') { %>prediction = model.predict(synthetic_input)
+print(f"Predicted rings: {prediction[0]:.1f}")
+<% } else if (framework === 'xgboost') { %>feature_names = ['Sex', 'Length', 'Diameter', 'Height', 'Whole_weight', 'Shucked_weight', 'Viscera_weight', 'Shell_weight']
+dtest = xgb.DMatrix(synthetic_input, feature_names=feature_names)
+prediction = model.predict(dtest)
+print(f"Predicted rings: {prediction[0]:.1f}")
+<% } else if (framework === 'tensorflow') { %>
+<% if (modelFormat === 'SavedModel') { %>prediction = model(synthetic_input)
+output_key = list(prediction.keys())[0]
+result = prediction[output_key].numpy()
+print(f"Prediction: {result[0][0]:.2f} rings")<% } else { %>prediction = model.predict(synthetic_input)
+print(f"Prediction: {prediction[0][0]:.2f} rings")<% } %>
+<% } %>
+<% } %>