@aws/ml-container-creator 0.2.0

package/templates/do/deploy

@@ -0,0 +1,1560 @@
+#!/bin/bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+set -u
+set -o pipefail
+
+# Parse flags
+FORCE_NEW=false
+FORCE_IC=false
+for arg in "$@"; do
+    case "$arg" in
+        --force) FORCE_NEW=true ;;
+        --force-ic) FORCE_IC=true ;;
+        --help|-h)
+            echo "Usage: ./do/deploy [--force] [--force-ic]"
+            echo ""
+            echo "Options:"
+            echo "  --force      Create a new endpoint and IC, even if one already exists."
+            echo "  --force-ic   Recreate just the IC on the existing endpoint."
+            echo ""
+            echo "Without flags, deploy resumes from the last run."
+            exit 0
+            ;;
+    esac
+done
+
+# Source configuration
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${SCRIPT_DIR}/config"
+
+echo "🚀 Deploying to AWS"
+echo "   Project: ${PROJECT_NAME}"
+echo "   Deployment config: ${DEPLOYMENT_CONFIG}"
+echo "   Region: ${AWS_REGION}"
+echo "   Build target: ${BUILD_TARGET}"
+echo "   Deployment target: ${DEPLOYMENT_TARGET}"
+<% if (deploymentTarget === 'managed-inference') { %>
+echo "   Instance type: ${INSTANCE_TYPE}"
+<% } else if (deploymentTarget === 'async-inference') { %>
+echo "   Instance type: ${INSTANCE_TYPE}"
+echo "   S3 output: ${ASYNC_S3_OUTPUT_PATH}"
+echo "   SNS success: ${ASYNC_SNS_SUCCESS_TOPIC}"
+echo "   SNS error: ${ASYNC_SNS_ERROR_TOPIC}"
+<% if (asyncMaxConcurrentInvocations) { %>
+echo "   Max concurrent: ${ASYNC_MAX_CONCURRENT_INVOCATIONS}"
+<% } %>
+<% } else if (deploymentTarget === 'hyperpod-eks') { %>
+echo "   HyperPod cluster: ${HYPERPOD_CLUSTER_NAME}"
+echo "   Namespace: ${HYPERPOD_NAMESPACE}"
+echo "   Replicas: ${HYPERPOD_REPLICAS}"
+<% } else if (deploymentTarget === 'batch-transform') { %>
+echo "   Instance type: ${INSTANCE_TYPE}"
+echo "   S3 input: ${BATCH_INPUT_PATH}"
+echo "   S3 output: ${BATCH_OUTPUT_PATH}"
+echo "   Instance count: ${BATCH_INSTANCE_COUNT}"
+echo "   Split type: ${BATCH_SPLIT_TYPE}"
+echo "   Strategy: ${BATCH_STRATEGY}"
+<% } %>
+
+# Check AWS credentials
+echo "🔍 Validating AWS credentials..."
+if ! aws sts get-caller-identity &> /dev/null; then
+    echo "❌ AWS credentials not configured"
+    echo "   Run: aws configure"
+    echo "   Or set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables"
+    exit 4
+fi
+
+AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+echo "✅ AWS credentials validated (Account: ${AWS_ACCOUNT_ID})"
+
+# Construct ECR repository URL
+ECR_REPOSITORY="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPOSITORY_NAME}"
+
+# ============================================================
+# Shared: Verify ECR image exists
+# ============================================================
+echo "🔍 Verifying ECR image exists..."
+if ! aws ecr describe-images \
+    --repository-name "${ECR_REPOSITORY_NAME}" \
+    --image-ids imageTag="${PROJECT_NAME}-latest" \
+    --region "${AWS_REGION}" &> /dev/null; then
+    
+    echo "❌ ECR image not found: ${ECR_REPOSITORY}:${PROJECT_NAME}-latest"
+    echo ""
+    echo "Please build and push your image first:"
+    echo "  ./do/submit"
+    echo ""
+    echo "After the build completes successfully, run this deploy script again."
+    exit 4
+fi
+
+echo "✅ ECR image found: ${ECR_REPOSITORY}:${PROJECT_NAME}-latest"
+IMAGE_TAG="${PROJECT_NAME}-latest"
+
+<% if (deploymentTarget === 'managed-inference') { %>
+# ============================================================
+# SageMaker Managed Inference Deployment (Inference Components)
+# ============================================================
+
+# Validate execution role ARN
+if [ -z "${ROLE_ARN:-}" ]; then
+    echo "❌ Execution role ARN not provided"
+    echo ""
+    echo "Usage:"
+    echo "  export ROLE_ARN=arn:aws:iam::ACCOUNT_ID:role/YOUR_ROLE"
+    echo "  ./do/deploy"
+    echo ""
+    echo "Or set ROLE_ARN in do/config"
+    echo ""
+    echo "The execution role must have permissions for:"
+    echo "  • SageMaker endpoint and inference component management"
+    echo "  • ECR image access"
+    echo "  • S3 access (if using model artifacts)"
+    echo "  • CloudWatch Logs"
+    exit 3
+fi
+
+echo "   Using execution role: ${ROLE_ARN}"
+
+# Helper: persist a variable to do/config so other scripts can use it
+_update_config_var() {
+    local var_name="$1" var_value="$2" config_file="${SCRIPT_DIR}/config"
+    if grep -q "^export ${var_name}=" "${config_file}" 2>/dev/null; then
+        sed -i.bak "s|^export ${var_name}=.*|export ${var_name}=\"${var_value}\"|" "${config_file}"
+        rm -f "${config_file}.bak"
+    else
+        echo "" >> "${config_file}"
+        echo "export ${var_name}=\"${var_value}\"" >> "${config_file}"
+    fi
+}
+
+# Helper: query a SageMaker resource status, returns empty string if not found
+_get_endpoint_status() {
+    aws sagemaker describe-endpoint \
+        --endpoint-name "$1" \
+        --region "${AWS_REGION}" \
+        --query EndpointStatus \
+        --output text 2>/dev/null || echo ""
+}
+
+_get_ic_status() {
+    aws sagemaker describe-inference-component \
+        --inference-component-name "$1" \
+        --region "${AWS_REGION}" \
+        --query InferenceComponentStatus \
+        --output text 2>/dev/null || echo ""
+}
+
+# Helper: find an InService IC on an endpoint (returns first match or empty)
+_find_active_ic_on_endpoint() {
+    aws sagemaker list-inference-components \
+        --endpoint-name "$1" \
+        --status-equals InService \
+        --region "${AWS_REGION}" \
+        --query 'InferenceComponents[0].InferenceComponentName' \
+        --output text 2>/dev/null || echo ""
+}
+
+# ============================================================
+# Idempotency: check for existing deployment from a previous run
+# ============================================================
+SKIP_TO=""
+
+if [ "${FORCE_NEW}" = true ]; then
+    echo "🔄 --force: ignoring previous deployment, creating new resources."
+elif [ "${FORCE_IC}" = true ] && [ -n "${ENDPOINT_NAME:-}" ]; then
+    EP_STATUS=$(_get_endpoint_status "${ENDPOINT_NAME}")
+    if [ "${EP_STATUS}" = "InService" ]; then
+        echo "🔄 --force-ic: recreating inference component on existing endpoint: ${ENDPOINT_NAME}"
+        SKIP_TO="create_ic"
+    else
+        echo "⚠️  --force-ic requires an InService endpoint, but ${ENDPOINT_NAME} is: ${EP_STATUS:-not found}"
+        echo "   Use --force to create a new endpoint, or wait for the current one."
+        exit 4
+    fi
+elif [ -n "${ENDPOINT_NAME:-}" ]; then
+    echo "🔍 Checking for existing deployment: ${ENDPOINT_NAME}"
+
+    EP_STATUS=$(_get_endpoint_status "${ENDPOINT_NAME}")
+
+    case "${EP_STATUS}" in
+        InService)
+            echo "✅ Endpoint already InService: ${ENDPOINT_NAME}"
+
+            # Check inference component
+            if [ -n "${INFERENCE_COMPONENT_NAME:-}" ]; then
+                IC_STATUS=$(_get_ic_status "${INFERENCE_COMPONENT_NAME}")
+
+                case "${IC_STATUS}" in
+                    InService)
+                        echo "✅ Inference component already InService: ${INFERENCE_COMPONENT_NAME}"
+                        echo ""
+                        echo "📋 Deployment is already live. Nothing to do."
+                        echo "   Endpoint: ${ENDPOINT_NAME}"
+                        echo "   Inference Component: ${INFERENCE_COMPONENT_NAME}"
+                        echo ""
+                        echo "🧪 Test your endpoint:"
+                        echo "   ./do/test"
+                        echo ""
+                        echo "🧹 Clean up when done:"
+                        echo "   ./do/clean endpoint"
+                        exit 0
+                        ;;
+                    Creating)
+                        echo "⏳ Inference component still creating: ${INFERENCE_COMPONENT_NAME}"
+                        SKIP_TO="wait_ic"
+                        IC_NAME="${INFERENCE_COMPONENT_NAME}"
+                        ;;
+                    Failed)
+                        echo "⚠️  Inference component failed: ${INFERENCE_COMPONENT_NAME}"
+                        echo "   Will create a new inference component on the existing endpoint."
+                        SKIP_TO="create_ic"
+                        ;;
+                    *)
+                        # Stored IC not found — check if a different IC is running on this endpoint
+                        LIVE_IC=$(_find_active_ic_on_endpoint "${ENDPOINT_NAME}")
+                        if [ -n "${LIVE_IC}" ] && [ "${LIVE_IC}" != "None" ]; then
+                            echo "✅ Found running inference component on endpoint: ${LIVE_IC}"
+                            echo "   (config had stale reference: ${INFERENCE_COMPONENT_NAME})"
+                            _update_config_var "INFERENCE_COMPONENT_NAME" "${LIVE_IC}"
+                            echo ""
+                            echo "📋 Deployment is already live. Nothing to do."
+                            echo "   Endpoint: ${ENDPOINT_NAME}"
+                            echo "   Inference Component: ${LIVE_IC}"
+                            echo ""
+                            echo "🧪 Test your endpoint:"
+                            echo "   ./do/test"
+                            echo ""
+                            echo "🧹 Clean up when done:"
+                            echo "   ./do/clean endpoint"
+                            exit 0
+                        else
+                            echo "   No existing inference component found on endpoint. Will create one."
+                            SKIP_TO="create_ic"
+                        fi
+                        ;;
+                esac
+            else
+                # No IC name in config — check if one is already running on the endpoint
+                LIVE_IC=$(_find_active_ic_on_endpoint "${ENDPOINT_NAME}")
+                if [ -n "${LIVE_IC}" ] && [ "${LIVE_IC}" != "None" ]; then
+                    echo "✅ Found running inference component on endpoint: ${LIVE_IC}"
+                    _update_config_var "INFERENCE_COMPONENT_NAME" "${LIVE_IC}"
+                    echo ""
+                    echo "📋 Deployment is already live. Nothing to do."
+                    echo "   Endpoint: ${ENDPOINT_NAME}"
+                    echo "   Inference Component: ${LIVE_IC}"
+                    echo ""
+                    echo "🧪 Test your endpoint:"
+                    echo "   ./do/test"
+                    echo ""
+                    echo "🧹 Clean up when done:"
+                    echo "   ./do/clean endpoint"
+                    exit 0
+                else
+                    SKIP_TO="create_ic"
+                fi
+            fi
+            ;;
+        Creating|Updating)
+            echo "⏳ Endpoint still ${EP_STATUS}: ${ENDPOINT_NAME}"
+            SKIP_TO="wait_endpoint"
+            ;;
+        Failed)
+            echo "⚠️  Previous endpoint failed: ${ENDPOINT_NAME}"
+            echo "   Creating a new deployment. Clean up the failed endpoint with:"
+            echo "   ./do/clean endpoint"
+            echo ""
+            # Fall through to create new resources
+            ;;
+        "")
+            echo "   Previous endpoint not found (may have been cleaned up). Creating new deployment."
+            ;;
+        *)
+            echo "   Endpoint in unexpected state: ${EP_STATUS}. Creating new deployment."
+            ;;
+    esac
+fi
+
+# ============================================================
+# Step 1: Create endpoint configuration (skip if resuming)
+# ============================================================
+if [ -z "${SKIP_TO}" ]; then
+    TIMESTAMP=$(date +%s)
+    ENDPOINT_CONFIG_NAME="${PROJECT_NAME}-epc-${TIMESTAMP}"
+    ENDPOINT_NAME="${PROJECT_NAME}-endpoint-${TIMESTAMP}"
+    IC_NAME="${PROJECT_NAME}-ic-${TIMESTAMP}"
+
+    _update_config_var "ENDPOINT_NAME" "${ENDPOINT_NAME}"
+    _update_config_var "ENDPOINT_CONFIG_NAME" "${ENDPOINT_CONFIG_NAME}"
+    _update_config_var "INFERENCE_COMPONENT_NAME" "${IC_NAME}"
+
+    # Build production variant JSON
+    VARIANT_JSON="[{\"VariantName\":\"AllTraffic\",\"InstanceType\":\"${INSTANCE_TYPE}\",\"InitialInstanceCount\":1"
+
+    if [ -n "${INFERENCE_AMI_VERSION:-}" ]; then
+        VARIANT_JSON="${VARIANT_JSON},\"InferenceAmiVersion\":\"${INFERENCE_AMI_VERSION}\""
+        echo "   AMI version: ${INFERENCE_AMI_VERSION}"
+    fi
+
+    VARIANT_JSON="${VARIANT_JSON}}]"
+
+    echo "⚙️  Creating endpoint configuration: ${ENDPOINT_CONFIG_NAME}"
+    if ! aws sagemaker create-endpoint-config \
+        --endpoint-config-name "${ENDPOINT_CONFIG_NAME}" \
+        --execution-role-arn "${ROLE_ARN}" \
+        --production-variants "${VARIANT_JSON}" \
+        --region "${AWS_REGION}"; then
+
+        echo "❌ Failed to create endpoint configuration"
+        echo "   Check that:"
+        echo "   • The execution role ARN is valid"
+        echo "   • The instance type is valid: ${INSTANCE_TYPE}"
+        echo "   • The instance type is available in region: ${AWS_REGION}"
+        echo "   • You have sufficient service quota for the instance type"
+        exit 4
+    fi
+
+    echo "✅ Endpoint configuration created: ${ENDPOINT_CONFIG_NAME}"
+
+    # Record endpoint config in manifest (non-blocking)
+    ENDPOINT_CONFIG_ARN="arn:aws:sagemaker:${AWS_REGION}:${AWS_ACCOUNT_ID}:endpoint-config/${ENDPOINT_CONFIG_NAME}"
+    ./do/manifest add \
+        --type sagemaker-endpoint-config \
+        --id "${ENDPOINT_CONFIG_ARN}" \
+        --project "${PROJECT_NAME}" \
+        --meta "{\"endpointConfigName\":\"${ENDPOINT_CONFIG_NAME}\",\"instanceType\":\"${INSTANCE_TYPE}\",\"region\":\"${AWS_REGION}\"}" \
+        2>/dev/null || true
+
+    # Step 2: Create endpoint
+    echo "🚀 Creating endpoint: ${ENDPOINT_NAME}"
+    if ! aws sagemaker create-endpoint \
+        --endpoint-name "${ENDPOINT_NAME}" \
+        --endpoint-config-name "${ENDPOINT_CONFIG_NAME}" \
+        --region "${AWS_REGION}"; then
+
+        echo "❌ Failed to create endpoint"
+        echo "   Check that:"
+        echo "   • Your IAM credentials have sagemaker:CreateEndpoint permission"
+        echo "   • You have sufficient service quota in region: ${AWS_REGION}"
+        exit 4
+    fi
+
+    echo "✅ Endpoint creation initiated: ${ENDPOINT_NAME}"
+
+    # Record endpoint in manifest (non-blocking)
+    ENDPOINT_ARN="arn:aws:sagemaker:${AWS_REGION}:${AWS_ACCOUNT_ID}:endpoint/${ENDPOINT_NAME}"
+    ./do/manifest add \
+        --type sagemaker-endpoint \
+        --id "${ENDPOINT_ARN}" \
+        --project "${PROJECT_NAME}" \
+        --meta "{\"endpointName\":\"${ENDPOINT_NAME}\",\"instanceType\":\"${INSTANCE_TYPE}\",\"region\":\"${AWS_REGION}\"}" \
+        2>/dev/null || true
+fi
+
+# ============================================================
+# Wait for endpoint (skip if already InService)
+# ============================================================
+if [ -z "${SKIP_TO}" ] || [ "${SKIP_TO}" = "wait_endpoint" ]; then
+    echo "⏳ Waiting for endpoint to reach InService status..."
+    echo "   This may take a few minutes..."
+    echo "   If this times out, re-run ./do/deploy to resume."
+
+    if ! aws sagemaker wait endpoint-in-service \
+        --endpoint-name "${ENDPOINT_NAME}" \
+        --region "${AWS_REGION}"; then
+
+        # Check if it was a credential expiration vs actual failure
+        EP_CHECK=$(_get_endpoint_status "${ENDPOINT_NAME}" 2>/dev/null)
+        if [ "${EP_CHECK}" = "Creating" ]; then
+            echo ""
+            echo "⚠️  Wait interrupted (credentials may have expired), but endpoint is still creating."
+            echo "   Refresh your credentials and re-run ./do/deploy to resume."
+            echo ""
+            echo "   Or check status manually:"
+            echo "   aws sagemaker describe-endpoint --endpoint-name ${ENDPOINT_NAME} --region ${AWS_REGION} --query EndpointStatus"
+            exit 4
+        fi
+
+        echo "❌ Endpoint failed to reach InService status"
+        echo "   Check CloudWatch Logs for details:"
+        echo "   https://console.aws.amazon.com/cloudwatch/home?region=${AWS_REGION}#logsV2:log-groups/log-group//aws/sagemaker/Endpoints/${ENDPOINT_NAME}"
+        exit 4
+    fi
+
+    echo "✅ Endpoint is InService: ${ENDPOINT_NAME}"
+fi
+
+# ============================================================
+# Step 3: Create inference component (skip if resuming from wait_ic)
+# ============================================================
+if [ -z "${SKIP_TO}" ] || [ "${SKIP_TO}" = "create_ic" ] || [ "${SKIP_TO}" = "wait_endpoint" ]; then
+    # Generate new IC name if resuming after endpoint wait or failed IC
+    if [ "${SKIP_TO}" = "create_ic" ] || [ "${SKIP_TO}" = "wait_endpoint" ]; then
+        TIMESTAMP=$(date +%s)
+        IC_NAME="${PROJECT_NAME}-ic-${TIMESTAMP}"
+        _update_config_var "INFERENCE_COMPONENT_NAME" "${IC_NAME}"
+    fi
+
+    echo "📦 Creating inference component: ${IC_NAME}"
+    if ! aws sagemaker create-inference-component \
+        --inference-component-name "${IC_NAME}" \
+        --endpoint-name "${ENDPOINT_NAME}" \
+        --variant-name "AllTraffic" \
+        --specification "{
+            \"Container\": {
+                \"Image\": \"${ECR_REPOSITORY}:${IMAGE_TAG}\"
+            },
+            \"StartupParameters\": {
+                \"ContainerStartupHealthCheckTimeoutInSeconds\": 900
+            },
+            \"ComputeResourceRequirements\": {
+                \"NumberOfAcceleratorDevicesRequired\": 1,
+                \"MinMemoryRequiredInMb\": 1024
+            }
+        }" \
+        --runtime-config "{\"CopyCount\": 1}" \
+        --region "${AWS_REGION}"; then
+
+        echo "❌ Failed to create inference component"
+        echo "   Check that:"
+        echo "   • The ECR image exists and is accessible"
+        echo "   • The endpoint is in InService status"
+        echo "   • The compute resource requirements fit the instance type: ${INSTANCE_TYPE}"
+        exit 4
+    fi
+
+    echo "✅ Inference component creation initiated: ${IC_NAME}"
+
+    # Record inference component in manifest (non-blocking)
+    IC_ARN="arn:aws:sagemaker:${AWS_REGION}:${AWS_ACCOUNT_ID}:inference-component/${IC_NAME}"
+    ./do/manifest add \
+        --type sagemaker-inference-component \
+        --id "${IC_ARN}" \
+        --project "${PROJECT_NAME}" \
+        --meta "{\"inferenceComponentName\":\"${IC_NAME}\",\"endpointName\":\"${ENDPOINT_NAME}\",\"instanceType\":\"${INSTANCE_TYPE}\",\"region\":\"${AWS_REGION}\"}" \
+        2>/dev/null || true
+fi
+
+# ============================================================
+# Wait for inference component
+# ============================================================
+echo "⏳ Waiting for inference component to reach InService status..."
+echo "   This may take 5-10 minutes..."
+echo "   If this times out, re-run ./do/deploy to resume."
+
+# Poll loop — replaces `aws sagemaker wait inference-component-in-service`
+# which is only available in AWS CLI v2.15+
+IC_WAIT_TIMEOUT=1800  # 30 minutes max
+IC_WAIT_START=$(date +%s)
+
+while true; do
+    IC_STATUS=$(_get_ic_status "${IC_NAME}" 2>/dev/null)
+
+    case "${IC_STATUS}" in
+        InService)
+            break
+            ;;
+        Failed)
+            echo "❌ Inference component failed to reach InService status"
+            echo "   Check CloudWatch Logs for details:"
+            echo "   https://console.aws.amazon.com/cloudwatch/home?region=${AWS_REGION}#logsV2:log-groups/log-group//aws/sagemaker/Endpoints/${ENDPOINT_NAME}"
+            echo ""
+            echo "   Debug:"
+            echo "   aws sagemaker describe-inference-component --inference-component-name ${IC_NAME} --region ${AWS_REGION}"
+            exit 4
+            ;;
+        Creating)
+            # Check timeout
+            IC_ELAPSED=$(( $(date +%s) - IC_WAIT_START ))
+            if [ "${IC_ELAPSED}" -ge "${IC_WAIT_TIMEOUT}" ]; then
+                echo ""
+                echo "⚠️  Inference component still creating after ${IC_WAIT_TIMEOUT}s."
+                echo "   Re-run ./do/deploy to resume waiting."
+                echo ""
+                echo "   Or check status manually:"
+                echo "   aws sagemaker describe-inference-component --inference-component-name ${IC_NAME} --region ${AWS_REGION}"
+                exit 4
+            fi
+            echo "   $(date +%H:%M:%S) Status: Creating (${IC_ELAPSED}s elapsed)..."
+            sleep 30
+            ;;
+        "")
+            echo "⚠️  Could not determine inference component status (credentials may have expired)."
+            echo "   Re-run ./do/deploy to resume."
+            exit 4
+            ;;
+        *)
+            echo "   $(date +%H:%M:%S) Status: ${IC_STATUS}..."
+            sleep 30
+            ;;
+    esac
+done
+
+echo "✅ Deployment complete!"
+echo ""
+echo "📋 Deployment Details:"
+echo "   Endpoint: ${ENDPOINT_NAME}"
+echo "   Endpoint Config: ${ENDPOINT_CONFIG_NAME}"
+echo "   Inference Component: ${IC_NAME}"
+echo "   Region: ${AWS_REGION}"
+echo "   Instance Type: ${INSTANCE_TYPE}"
+echo "   Image: ${ECR_REPOSITORY}:${IMAGE_TAG}"
+echo ""
+echo "🧪 Test your endpoint:"
+echo "   ./do/test"
+echo ""
+echo "📝 Register this deployment:"
+echo "   ./do/register"
+echo ""
+echo "🔍 Monitor your deployment:"
+echo "   aws sagemaker describe-inference-component --inference-component-name ${IC_NAME} --region ${AWS_REGION}"
+echo "   aws sagemaker describe-endpoint --endpoint-name ${ENDPOINT_NAME} --region ${AWS_REGION}"
+echo ""
+echo "🧹 Clean up when done:"
+echo "   ./do/clean endpoint"
+
+<% } else if (deploymentTarget === 'async-inference') { %>
+# ============================================================
+# SageMaker Managed Inference - Async Deployment (Model-Based)
+# SageMaker async inference does NOT support Inference Components.
+# Flow: create-model → create-endpoint-config (with AsyncInferenceConfig) → create-endpoint
+# ============================================================
+
+# Validate execution role ARN
+if [ -z "${ROLE_ARN:-}" ]; then
+    echo "❌ Execution role ARN not provided"
+    echo ""
+    echo "Usage:"
+    echo "  export ROLE_ARN=arn:aws:iam::ACCOUNT_ID:role/YOUR_ROLE"
+    echo "  ./do/deploy"
+    echo ""
+    echo "Or set ROLE_ARN in do/config"
+    echo ""
+    echo "The execution role must have permissions for:"
+    echo "  • SageMaker model and endpoint management"
+    echo "  • ECR image access"
+    echo "  • S3 write access for async output path: ${ASYNC_S3_OUTPUT_PATH}"
+    echo "  • SNS publish permissions (optional, for notifications)"
+    echo "  • CloudWatch Logs"
+    exit 3
+fi
+
+echo "   Using execution role: ${ROLE_ARN}"
+
+# ============================================================
+# Bootstrap async infrastructure (S3 bucket + SNS topics)
+# ============================================================
+
+# Extract bucket name from S3 output path
+ASYNC_S3_BUCKET=$(echo "${ASYNC_S3_OUTPUT_PATH}" | sed 's|s3://||' | cut -d'/' -f1)
+
+<% if (!asyncS3OutputPath) { %>
+# Bootstrap default S3 bucket (check-and-create)
+echo "🔍 Checking if S3 bucket exists: ${ASYNC_S3_BUCKET}"
+if ! aws s3api head-bucket --bucket "${ASYNC_S3_BUCKET}" --region "${AWS_REGION}" 2>/dev/null; then
+    echo "📦 Creating S3 bucket: ${ASYNC_S3_BUCKET}"
+    if [ "${AWS_REGION}" = "us-east-1" ]; then
+        if ! aws s3api create-bucket \
+            --bucket "${ASYNC_S3_BUCKET}" \
+            --region "${AWS_REGION}"; then
+            echo "❌ Failed to create S3 bucket: ${ASYNC_S3_BUCKET}"
+            echo ""
+            echo "   Check that:"
+            echo "   • Your IAM credentials have s3:CreateBucket permission"
+            echo "   • The bucket name is not already taken globally"
+            exit 4
+        fi
+    else
+        if ! aws s3api create-bucket \
+            --bucket "${ASYNC_S3_BUCKET}" \
+            --region "${AWS_REGION}" \
+            --create-bucket-configuration LocationConstraint="${AWS_REGION}"; then
+            echo "❌ Failed to create S3 bucket: ${ASYNC_S3_BUCKET}"
+            echo ""
+            echo "   Check that:"
+            echo "   • Your IAM credentials have s3:CreateBucket permission"
+            echo "   • The bucket name is not already taken globally"
+            exit 4
+        fi
+    fi
+    echo "✅ S3 bucket created: ${ASYNC_S3_BUCKET}"
+else
+    echo "✅ S3 bucket exists: ${ASYNC_S3_BUCKET}"
+fi
+<% } else { %>
+# Custom S3 output path provided — skip bucket creation
+echo "✅ Using custom S3 output path: ${ASYNC_S3_OUTPUT_PATH}"
+<% } %>
+
+# Extract topic name from SNS success topic ARN
+ASYNC_SNS_SUCCESS_TOPIC_NAME=$(echo "${ASYNC_SNS_SUCCESS_TOPIC}" | awk -F: '{print $NF}')
+
+<% if (!asyncSnsSuccessTopic) { %>
+# Bootstrap default SNS success topic (check-and-create)
+echo "🔍 Checking if SNS success topic exists: ${ASYNC_SNS_SUCCESS_TOPIC_NAME}"
+if ! aws sns get-topic-attributes --topic-arn "${ASYNC_SNS_SUCCESS_TOPIC}" --region "${AWS_REGION}" 2>/dev/null; then
+    echo "📦 Creating SNS success topic: ${ASYNC_SNS_SUCCESS_TOPIC_NAME}"
+    if ! aws sns create-topic \
+        --name "${ASYNC_SNS_SUCCESS_TOPIC_NAME}" \
+        --region "${AWS_REGION}" > /dev/null; then
+        echo "❌ Failed to create SNS success topic: ${ASYNC_SNS_SUCCESS_TOPIC_NAME}"
+        echo ""
+        echo "   Check that:"
+        echo "   • Your IAM credentials have sns:CreateTopic permission"
+        exit 4
+    fi
+    echo "✅ SNS success topic created: ${ASYNC_SNS_SUCCESS_TOPIC_NAME}"
+else
+    echo "✅ SNS success topic exists: ${ASYNC_SNS_SUCCESS_TOPIC_NAME}"
+fi
+
+# Record SNS success topic in manifest (non-blocking)
+./do/manifest add \
+    --type sns-topic \
+    --id "${ASYNC_SNS_SUCCESS_TOPIC}" \
+    --project "${PROJECT_NAME}" \
+    --meta "{\"topicName\":\"${ASYNC_SNS_SUCCESS_TOPIC_NAME}\",\"purpose\":\"async-success\",\"region\":\"${AWS_REGION}\"}" \
+    2>/dev/null || true
+
+<% } else { %>
+# Custom SNS success topic ARN provided — skip topic creation
+echo "✅ Using custom SNS success topic: ${ASYNC_SNS_SUCCESS_TOPIC}"
+
+# Record SNS success topic in manifest (non-blocking)
+ASYNC_SNS_SUCCESS_TOPIC_NAME=$(echo "${ASYNC_SNS_SUCCESS_TOPIC}" | awk -F: '{print $NF}')
+./do/manifest add \
+    --type sns-topic \
+    --id "${ASYNC_SNS_SUCCESS_TOPIC}" \
+    --project "${PROJECT_NAME}" \
+    --meta "{\"topicName\":\"${ASYNC_SNS_SUCCESS_TOPIC_NAME}\",\"purpose\":\"async-success\",\"region\":\"${AWS_REGION}\"}" \
+    2>/dev/null || true
+
+<% } %>
+
+# Extract topic name from SNS error topic ARN
+ASYNC_SNS_ERROR_TOPIC_NAME=$(echo "${ASYNC_SNS_ERROR_TOPIC}" | awk -F: '{print $NF}')
+
+<% if (!asyncSnsErrorTopic) { %>
+# Bootstrap default SNS error topic (check-and-create)
+echo "🔍 Checking if SNS error topic exists: ${ASYNC_SNS_ERROR_TOPIC_NAME}"
+if ! aws sns get-topic-attributes --topic-arn "${ASYNC_SNS_ERROR_TOPIC}" --region "${AWS_REGION}" 2>/dev/null; then
+    echo "📦 Creating SNS error topic: ${ASYNC_SNS_ERROR_TOPIC_NAME}"
+    if ! aws sns create-topic \
+        --name "${ASYNC_SNS_ERROR_TOPIC_NAME}" \
+        --region "${AWS_REGION}" > /dev/null; then
+        echo "❌ Failed to create SNS error topic: ${ASYNC_SNS_ERROR_TOPIC_NAME}"
+        echo ""
+        echo "   Check that:"
+        echo "   • Your IAM credentials have sns:CreateTopic permission"
+        exit 4
+    fi
+    echo "✅ SNS error topic created: ${ASYNC_SNS_ERROR_TOPIC_NAME}"
+else
+    echo "✅ SNS error topic exists: ${ASYNC_SNS_ERROR_TOPIC_NAME}"
+fi
+
+# Record SNS error topic in manifest (non-blocking)
+./do/manifest add \
+    --type sns-topic \
+    --id "${ASYNC_SNS_ERROR_TOPIC}" \
+    --project "${PROJECT_NAME}" \
+    --meta "{\"topicName\":\"${ASYNC_SNS_ERROR_TOPIC_NAME}\",\"purpose\":\"async-error\",\"region\":\"${AWS_REGION}\"}" \
+    2>/dev/null || true
+
+<% } else { %>
+# Custom SNS error topic ARN provided — skip topic creation
+echo "✅ Using custom SNS error topic: ${ASYNC_SNS_ERROR_TOPIC}"
+
+# Record SNS error topic in manifest (non-blocking)
+ASYNC_SNS_ERROR_TOPIC_NAME=$(echo "${ASYNC_SNS_ERROR_TOPIC}" | awk -F: '{print $NF}')
+./do/manifest add \
+    --type sns-topic \
+    --id "${ASYNC_SNS_ERROR_TOPIC}" \
+    --project "${PROJECT_NAME}" \
+    --meta "{\"topicName\":\"${ASYNC_SNS_ERROR_TOPIC_NAME}\",\"purpose\":\"async-error\",\"region\":\"${AWS_REGION}\"}" \
+    2>/dev/null || true
+
+<% } %>
+
+# ============================================================
+# Create async endpoint (classic model-based flow)
+# SageMaker async inference does NOT support Inference Components.
+# Flow: create-model → create-endpoint-config (with AsyncInferenceConfig) → create-endpoint
+# ============================================================
+
+# Helper: persist a variable to do/config so other scripts can use it
+_update_config_var() {
+    local var_name="$1" var_value="$2" config_file="${SCRIPT_DIR}/config"
+    if grep -q "^export ${var_name}=" "${config_file}" 2>/dev/null; then
+        sed -i.bak "s|^export ${var_name}=.*|export ${var_name}=\"${var_value}\"|" "${config_file}"
+        rm -f "${config_file}.bak"
+    else
+        echo "" >> "${config_file}"
+        echo "export ${var_name}=\"${var_value}\"" >> "${config_file}"
+    fi
+}
+
+# Helper: query a SageMaker resource status, returns empty string if not found
+_get_endpoint_status() {
+    aws sagemaker describe-endpoint \
+        --endpoint-name "$1" \
+        --region "${AWS_REGION}" \
+        --query EndpointStatus \
+        --output text 2>/dev/null || echo ""
+}
+
+# ============================================================
+# Idempotency: check for existing deployment from a previous run
+# ============================================================
+SKIP_TO=""
+
+if [ "${FORCE_NEW}" = true ]; then
+    echo "🔄 --force: ignoring previous deployment, creating new resources."
+elif [ -n "${ENDPOINT_NAME:-}" ]; then
+    echo "🔍 Checking for existing deployment: ${ENDPOINT_NAME}"
+
+    EP_STATUS=$(_get_endpoint_status "${ENDPOINT_NAME}")
+
+    case "${EP_STATUS}" in
+        InService)
+            echo "✅ Async endpoint already InService: ${ENDPOINT_NAME}"
+            echo ""
+            echo "📋 Deployment is already live. Nothing to do."
+            echo "   Endpoint: ${ENDPOINT_NAME}"
+            echo ""
+            echo "🧪 Test your async endpoint:"
+            echo "   ./do/test"
+            echo ""
+            echo "🧹 Clean up when done:"
+            echo "   ./do/clean endpoint"
+            exit 0
+            ;;
+        Creating|Updating)
+            echo "⏳ Endpoint still ${EP_STATUS}: ${ENDPOINT_NAME}"
+            SKIP_TO="wait_endpoint"
+            ;;
+        Failed)
+            echo "⚠️  Previous endpoint failed: ${ENDPOINT_NAME}"
+            echo "   Creating a new deployment. Clean up the failed endpoint with:"
+            echo "   ./do/clean endpoint"
+            echo ""
+            ;;
+        "")
+            echo "   Previous endpoint not found (may have been cleaned up). Creating new deployment."
+            ;;
+        *)
+            echo "   Endpoint in unexpected state: ${EP_STATUS}. Creating new deployment."
+            ;;
+    esac
+fi
+
+# ============================================================
+# Create async resources (skip if resuming from wait)
+# ============================================================
+if [ -z "${SKIP_TO}" ]; then
+    TIMESTAMP=$(date +%s)
+    MODEL_NAME_SM="${PROJECT_NAME}-async-model-${TIMESTAMP}"
+    ENDPOINT_CONFIG_NAME="${PROJECT_NAME}-async-epc-${TIMESTAMP}"
+    ENDPOINT_NAME="${PROJECT_NAME}-async-ep-${TIMESTAMP}"
+
+    _update_config_var "ENDPOINT_NAME" "${ENDPOINT_NAME}"
+    _update_config_var "ENDPOINT_CONFIG_NAME" "${ENDPOINT_CONFIG_NAME}"
+    _update_config_var "SAGEMAKER_MODEL_NAME" "${MODEL_NAME_SM}"
+
+    # Step 1: Create SageMaker model
+    echo "📦 Creating SageMaker model: ${MODEL_NAME_SM}"
+    if ! aws sagemaker create-model \
+        --model-name "${MODEL_NAME_SM}" \
+        --primary-container "{\"Image\":\"${ECR_REPOSITORY}:${IMAGE_TAG}\"}" \
+        --execution-role-arn "${ROLE_ARN}" \
+        --region "${AWS_REGION}"; then
+
+        echo "❌ Failed to create SageMaker model"
+        echo "   Check that:"
+        echo "   • The execution role ARN is valid"
+        echo "   • The ECR image exists and is accessible"
+        echo "   • The IAM role has ecr:GetDownloadUrlForLayer permission"
+        exit 4
+    fi
+
+    echo "✅ SageMaker model created: ${MODEL_NAME_SM}"
+
+    # Record model in manifest (non-blocking)
+    MODEL_ARN="arn:aws:sagemaker:${AWS_REGION}:${AWS_ACCOUNT_ID}:model/${MODEL_NAME_SM}"
+    ./do/manifest add \
+        --type sagemaker-model \
+        --id "${MODEL_ARN}" \
+        --project "${PROJECT_NAME}" \
+        --meta "{\"modelName\":\"${MODEL_NAME_SM}\",\"region\":\"${AWS_REGION}\"}" \
+        2>/dev/null || true
+
+    # Build production variant JSON (classic: includes ModelName, no execution-role-arn on endpoint config)
+    VARIANT_JSON="[{\"VariantName\":\"AllTraffic\",\"ModelName\":\"${MODEL_NAME_SM}\",\"InstanceType\":\"${INSTANCE_TYPE}\",\"InitialInstanceCount\":1"
+
+    # Append InferenceAmiVersion if configured
+    if [ -n "${INFERENCE_AMI_VERSION:-}" ]; then
+        VARIANT_JSON="${VARIANT_JSON},\"InferenceAmiVersion\":\"${INFERENCE_AMI_VERSION}\""
+        echo "   AMI version: ${INFERENCE_AMI_VERSION}"
+    fi
+
+    VARIANT_JSON="${VARIANT_JSON}}]"
+
+    # Build AsyncInferenceConfig JSON
+    ASYNC_CONFIG="{\"OutputConfig\":{\"S3OutputPath\":\"${ASYNC_S3_OUTPUT_PATH}\",\"NotificationConfig\":{\"SuccessTopic\":\"${ASYNC_SNS_SUCCESS_TOPIC}\",\"ErrorTopic\":\"${ASYNC_SNS_ERROR_TOPIC}\"}}"
+
+    if [ -n "${ASYNC_MAX_CONCURRENT_INVOCATIONS:-}" ]; then
+        ASYNC_CONFIG="${ASYNC_CONFIG},\"ClientConfig\":{\"MaxConcurrentInvocationsPerInstance\":${ASYNC_MAX_CONCURRENT_INVOCATIONS}}"
+    fi
+
+    ASYNC_CONFIG="${ASYNC_CONFIG}}"
+
+    # Step 2: Create endpoint configuration with AsyncInferenceConfig (no --execution-role-arn)
+    echo "⚙️  Creating async endpoint configuration: ${ENDPOINT_CONFIG_NAME}"
+    if ! aws sagemaker create-endpoint-config \
+        --endpoint-config-name "${ENDPOINT_CONFIG_NAME}" \
+        --production-variants "${VARIANT_JSON}" \
+        --async-inference-config "${ASYNC_CONFIG}" \
+        --region "${AWS_REGION}"; then
+
+        echo "❌ Failed to create async endpoint configuration"
+        echo "   Check that:"
+        echo "   • The S3 output path is accessible: ${ASYNC_S3_OUTPUT_PATH}"
+        echo "   • The IAM role has s3:PutObject permission on the output path"
+        echo "   • The instance type is valid: ${INSTANCE_TYPE}"
+        echo "   • The instance type is available in region: ${AWS_REGION}"
+        echo "   • You have sufficient service quota for the instance type"
+        exit 4
+    fi
+
+    echo "✅ Async endpoint configuration created: ${ENDPOINT_CONFIG_NAME}"
+
+    # Record endpoint config in manifest (non-blocking)
+    ENDPOINT_CONFIG_ARN="arn:aws:sagemaker:${AWS_REGION}:${AWS_ACCOUNT_ID}:endpoint-config/${ENDPOINT_CONFIG_NAME}"
+    ./do/manifest add \
+        --type sagemaker-endpoint-config \
+        --id "${ENDPOINT_CONFIG_ARN}" \
+        --project "${PROJECT_NAME}" \
+        --meta "{\"endpointConfigName\":\"${ENDPOINT_CONFIG_NAME}\",\"instanceType\":\"${INSTANCE_TYPE}\",\"region\":\"${AWS_REGION}\"}" \
+        2>/dev/null || true
+
+    # Step 3: Create endpoint
+    echo "🚀 Creating async endpoint: ${ENDPOINT_NAME}"
+    if ! aws sagemaker create-endpoint \
+        --endpoint-name "${ENDPOINT_NAME}" \
+        --endpoint-config-name "${ENDPOINT_CONFIG_NAME}" \
+        --region "${AWS_REGION}"; then
+
+        echo "❌ Failed to create async endpoint"
+        echo "   Check that:"
+        echo "   • Your IAM credentials have sagemaker:CreateEndpoint permission"
+        echo "   • You have sufficient service quota in region: ${AWS_REGION}"
+        exit 4
+    fi
+
+    echo "✅ Async endpoint creation initiated: ${ENDPOINT_NAME}"
+
+    # Record endpoint in manifest (non-blocking)
+    ENDPOINT_ARN="arn:aws:sagemaker:${AWS_REGION}:${AWS_ACCOUNT_ID}:endpoint/${ENDPOINT_NAME}"
+    ./do/manifest add \
+        --type sagemaker-endpoint \
+        --id "${ENDPOINT_ARN}" \
+        --project "${PROJECT_NAME}" \
+        --meta "{\"endpointName\":\"${ENDPOINT_NAME}\",\"instanceType\":\"${INSTANCE_TYPE}\",\"region\":\"${AWS_REGION}\"}" \
+        2>/dev/null || true
+fi
+
+# ============================================================
+# Wait for endpoint (skip if already InService)
+# ============================================================
+if [ -z "${SKIP_TO}" ] || [ "${SKIP_TO}" = "wait_endpoint" ]; then
+    echo "⏳ Waiting for async endpoint to reach InService status..."
+    echo "   This may take several minutes..."
+    echo "   If this times out, re-run ./do/deploy to resume."
+
+    if ! aws sagemaker wait endpoint-in-service \
+        --endpoint-name "${ENDPOINT_NAME}" \
+        --region "${AWS_REGION}"; then
+
+        # Check if it was a credential expiration vs actual failure
+        EP_CHECK=$(_get_endpoint_status "${ENDPOINT_NAME}" 2>/dev/null)
+        if [ "${EP_CHECK}" = "Creating" ]; then
+            echo ""
+            echo "⚠️  Wait interrupted (credentials may have expired), but endpoint is still creating."
+            echo "   Refresh your credentials and re-run ./do/deploy to resume."
+            echo ""
+            echo "   Or check status manually:"
+            echo "   aws sagemaker describe-endpoint --endpoint-name ${ENDPOINT_NAME} --region ${AWS_REGION} --query EndpointStatus"
+            exit 4
+        fi
+
+        echo "❌ Async endpoint failed to reach InService status"
+        echo "   Check CloudWatch Logs for details:"
+        echo "   https://console.aws.amazon.com/cloudwatch/home?region=${AWS_REGION}#logsV2:log-groups/log-group//aws/sagemaker/Endpoints/${ENDPOINT_NAME}"
+        exit 4
+    fi
+fi
+
+echo "✅ Async deployment complete!"
+echo ""
+echo "📋 Deployment Details:"
+echo "   Endpoint: ${ENDPOINT_NAME}"
+echo "   Endpoint Config: ${ENDPOINT_CONFIG_NAME}"
+echo "   Model: ${MODEL_NAME_SM}"
+echo "   Region: ${AWS_REGION}"
+echo "   Instance Type: ${INSTANCE_TYPE}"
+echo "   Image: ${ECR_REPOSITORY}:${IMAGE_TAG}"
+echo "   S3 Output: ${ASYNC_S3_OUTPUT_PATH}"
+echo "   SNS Success: ${ASYNC_SNS_SUCCESS_TOPIC}"
+echo "   SNS Error: ${ASYNC_SNS_ERROR_TOPIC}"
+echo ""
+echo "🧪 Test your async endpoint:"
+echo "   ./do/test"
+echo ""
+echo "📝 Register this deployment:"
+echo "   ./do/register"
+echo ""
+echo "🔍 Monitor your deployment:"
+echo "   aws sagemaker describe-endpoint --endpoint-name ${ENDPOINT_NAME} --region ${AWS_REGION}"
+echo ""
+echo "🧹 Clean up when done:"
+echo "   ./do/clean endpoint"
+
+<% } else if (deploymentTarget === 'hyperpod-eks') { %>
+# ============================================================
+# HyperPod EKS Deployment
+# ============================================================
+
+# Get kubeconfig for HyperPod cluster
+echo "🔑 Configuring kubectl for HyperPod cluster..."
+KUBECONFIG_PATH="${HOME}/.kube/hyperpod-${HYPERPOD_CLUSTER_NAME}"
+
+# Step 1: Describe the HyperPod cluster to get the underlying EKS cluster ARN
+EKS_CLUSTER_ARN=$(aws sagemaker describe-cluster \
+    --cluster-name "${HYPERPOD_CLUSTER_NAME}" \
+    --region "${AWS_REGION}" \
+    --query "Orchestrator.Eks.ClusterArn" \
+    --output text 2>&1) || {
+    echo "❌ Failed to describe HyperPod cluster: ${HYPERPOD_CLUSTER_NAME}"
+    echo ""
+    echo "   Error details:"
+    echo "   ${EKS_CLUSTER_ARN}"
+    echo ""
+    echo "   Check that:"
+    echo "   • The cluster name is correct"
+    echo "   • The cluster exists in region: ${AWS_REGION}"
+    echo "   • Your IAM user/role has permission to access the cluster"
+    echo ""
+    echo "   Required IAM permissions:"
+    echo "   • sagemaker:DescribeCluster"
+    echo "   • eks:DescribeCluster"
+    exit 4
+}
+
+# Step 2: Extract the EKS cluster name from the ARN
+EKS_CLUSTER_NAME=$(echo "${EKS_CLUSTER_ARN}" | awk -F'/' '{print $NF}')
+echo "   HyperPod cluster: ${HYPERPOD_CLUSTER_NAME}"
+echo "   EKS cluster: ${EKS_CLUSTER_NAME}"
+
+# Step 3: Update kubeconfig using the EKS cluster
+if ! aws eks update-kubeconfig \
+    --name "${EKS_CLUSTER_NAME}" \
+    --region "${AWS_REGION}" \
+    --kubeconfig "${KUBECONFIG_PATH}" 2>&1; then
+    echo "❌ Failed to configure kubectl for EKS cluster: ${EKS_CLUSTER_NAME}"
+    echo ""
+    echo "   Required IAM permissions:"
+    echo "   • eks:DescribeCluster"
+    echo "   • eks:AccessKubernetesApi"
+    exit 4
+fi
+
+export KUBECONFIG="${KUBECONFIG_PATH}"
+echo "✅ Kubeconfig saved to: ${KUBECONFIG_PATH}"
+
+# Verify cluster connectivity
+echo "🔍 Verifying cluster connectivity..."
+if ! kubectl cluster-info &> /dev/null; then
+    echo "❌ Cannot connect to HyperPod cluster"
+    echo ""
+    echo "   Check that:"
+    echo "   • The cluster is in 'InService' status"
+    echo "   • Your network can reach the cluster API server"
+    echo "   • Your IAM credentials are valid"
+    exit 4
+fi
+echo "✅ Connected to HyperPod cluster"
+
+# Create namespace if it doesn't exist
+echo "📁 Ensuring namespace exists: ${HYPERPOD_NAMESPACE}"
+if ! kubectl create namespace "${HYPERPOD_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - 2>&1; then
+    echo "⚠️  Warning: Could not create/verify namespace"
+fi
+
+# Apply Kubernetes manifests
+echo "📄 Applying Kubernetes manifests from hyperpod/..."
+
+# Substitute shell variables (e.g. ${AWS_ACCOUNT_ID}) in manifests before applying
+export AWS_ACCOUNT_ID
+export ECR_IMAGE="${ECR_REPOSITORY}:${IMAGE_TAG}"
+
+APPLY_OUTPUT=""
+APPLY_EXIT_CODE=0
+for manifest in hyperpod/*.yaml; do
+    # Skip files that contain no Kubernetes objects (e.g. comment-only PVC stubs)
+    RENDERED=$(envsubst < "${manifest}")
+    if echo "${RENDERED}" | grep -q '^kind:'; then
+        FILE_OUTPUT=$(echo "${RENDERED}" | kubectl apply -n "${HYPERPOD_NAMESPACE}" -f - 2>&1) || {
+            APPLY_EXIT_CODE=$?
+        }
+        APPLY_OUTPUT="${APPLY_OUTPUT}${FILE_OUTPUT}\n"
+    fi
+done
+
+if [ "${APPLY_EXIT_CODE}" -ne 0 ]; then
+    echo ""
+    echo "❌ Failed to apply Kubernetes manifests"
+    echo ""
+    echo "   Error details:"
+    echo "   ${APPLY_OUTPUT}"
+    echo ""
+    echo "   Common issues:"
+    echo "   • Insufficient node capacity - check available GPU nodes"
+    echo "   • Resource requests exceed node capacity"
+    echo "   • RBAC permissions - ensure you have permission to create resources in namespace '${HYPERPOD_NAMESPACE}'"
+    echo "   • Invalid manifest syntax"
+<% if (fsxVolumeHandle) { %>
+    echo "   • PVC creation failure - verify the FSx CSI driver is installed on the cluster"
+    echo "     kubectl get csidriver -o name | grep fsx"
+<% } %>
+    echo ""
+    echo "   Debug commands:"
+    echo "   kubectl get nodes -o wide"
+    echo "   kubectl describe nodes"
+    echo "   kubectl get events -n ${HYPERPOD_NAMESPACE}"
+    exit ${APPLY_EXIT_CODE}
+fi
+
+echo "✅ Kubernetes manifests applied"
+
+# Record k8s deployment and service in manifest (non-blocking)
+./do/manifest add \
+    --type k8s-deployment \
+    --id "${HYPERPOD_NAMESPACE}/${PROJECT_NAME}" \
+    --project "${PROJECT_NAME}" \
+    --meta "{\"namespace\":\"${HYPERPOD_NAMESPACE}\",\"deploymentName\":\"${PROJECT_NAME}\",\"clusterName\":\"${HYPERPOD_CLUSTER_NAME}\",\"region\":\"${AWS_REGION}\"}" \
+    2>/dev/null || true
+
+./do/manifest add \
+    --type k8s-service \
+    --id "${HYPERPOD_NAMESPACE}/${PROJECT_NAME}" \
+    --project "${PROJECT_NAME}" \
+    --meta "{\"namespace\":\"${HYPERPOD_NAMESPACE}\",\"serviceName\":\"${PROJECT_NAME}\",\"clusterName\":\"${HYPERPOD_CLUSTER_NAME}\",\"region\":\"${AWS_REGION}\"}" \
+    2>/dev/null || true
+
+# Wait for deployment to be ready
+DEPLOY_TIMEOUT=${DEPLOY_TIMEOUT:-1200}
+echo "⏳ Waiting for deployment to be ready (timeout: ${DEPLOY_TIMEOUT}s)..."
+echo "   This may take several minutes for GPU workloads..."
+echo ""
+
+# Poll pod status every 30s while rollout is in progress
+(
+    while true; do
+        sleep 30
+        POD_STATUS=$(kubectl get pods -n "${HYPERPOD_NAMESPACE}" -l app=${PROJECT_NAME} \
+            --no-headers 2>/dev/null | head -5)
+        if [ -n "${POD_STATUS}" ]; then
+            echo "   📊 $(date +%H:%M:%S) Pod status:"
+            echo "${POD_STATUS}" | while read -r line; do echo "      ${line}"; done
+        fi
+    done
+) &
+STATUS_PID=$!
+trap "kill ${STATUS_PID} 2>/dev/null; wait ${STATUS_PID} 2>/dev/null" EXIT
+
+ROLLOUT_OUTPUT=$(kubectl rollout status deployment/${PROJECT_NAME} -n "${HYPERPOD_NAMESPACE}" --timeout=${DEPLOY_TIMEOUT}s 2>&1) || {
+    ROLLOUT_EXIT_CODE=$?
+    kill ${STATUS_PID} 2>/dev/null
+    echo ""
+    echo "❌ Deployment failed to become ready within timeout"
+    echo ""
+    echo "   Error details:"
+    echo "   ${ROLLOUT_OUTPUT}"
+    echo ""
+    echo "   Current pod state:"
+    kubectl get pods -n "${HYPERPOD_NAMESPACE}" -l app=${PROJECT_NAME} -o wide 2>/dev/null
+    echo ""
+    echo "   Debug commands:"
+    echo "   kubectl describe pods -n ${HYPERPOD_NAMESPACE} -l app=${PROJECT_NAME}"
+    echo "   kubectl logs -n ${HYPERPOD_NAMESPACE} -l app=${PROJECT_NAME} --tail=100"
+    echo ""
+    echo "   Common issues:"
+    echo "   • Image pull errors - check ECR permissions"
+    echo "   • Resource scheduling - insufficient GPU nodes"
+    echo "   • Container crash - check application logs"
+<% if (fsxVolumeHandle) { %>
+    echo "   • PVC binding errors - verify FSx CSI driver is installed on the cluster"
+    echo "     kubectl get pvc -n ${HYPERPOD_NAMESPACE}"
+    echo "     kubectl describe pvc -n ${HYPERPOD_NAMESPACE}"
+    echo "     kubectl get csidriver -o name | grep fsx"
+<% } %>
+    exit ${ROLLOUT_EXIT_CODE}
+}
+
+kill ${STATUS_PID} 2>/dev/null
+wait ${STATUS_PID} 2>/dev/null
+
+echo "✅ HyperPod EKS deployment complete!"
+echo ""
+echo "📋 Deployment Details:"
+echo "   Cluster: ${HYPERPOD_CLUSTER_NAME}"
+echo "   Namespace: ${HYPERPOD_NAMESPACE}"
+echo "   Deployment: ${PROJECT_NAME}"
+echo "   Replicas: ${HYPERPOD_REPLICAS}"
+echo "   Image: ${ECR_REPOSITORY}:${IMAGE_TAG}"
+echo ""
+echo "🔍 Check deployment status:"
+echo "   export KUBECONFIG=${KUBECONFIG_PATH}"
+echo "   kubectl get pods -n ${HYPERPOD_NAMESPACE}"
+echo "   kubectl get svc -n ${HYPERPOD_NAMESPACE}"
+echo ""
+echo "🧪 Test your deployment:"
+echo "   ./do/test"
+echo ""
+echo "📝 Register this deployment:"
+echo "   ./do/register"
+echo ""
+echo "📋 View logs:"
+echo "   ./do/logs"
+echo ""
+echo "🧹 Clean up when done:"
+echo "   ./do/clean hyperpod"
+
+# Write kubeconfig path to config so other scripts can use it (idempotent)
+_update_config_var() {
+    local var_name="$1" var_value="$2" config_file="${SCRIPT_DIR}/config"
+    if grep -q "^export ${var_name}=" "${config_file}" 2>/dev/null; then
+        sed -i.bak "s|^export ${var_name}=.*|export ${var_name}=\"${var_value}\"|" "${config_file}"
+        rm -f "${config_file}.bak"
+    else
+        echo "" >> "${config_file}"
+        echo "export ${var_name}=\"${var_value}\"" >> "${config_file}"
+    fi
+}
+
+_update_config_var "KUBECONFIG" "${KUBECONFIG_PATH}"
+
+<% } else if (deploymentTarget === 'batch-transform') { %>
+# ============================================================
+# SageMaker Managed Inference - Batch Deployment
+# Flow: create-model → create-transform-job → poll until completion
+# ============================================================
+
+# Validate execution role ARN
+if [ -z "${ROLE_ARN:-}" ]; then
+    echo "❌ Execution role ARN not provided"
+    echo ""
+    echo "Usage:"
+    echo "  export ROLE_ARN=arn:aws:iam::ACCOUNT_ID:role/YOUR_ROLE"
+    echo "  ./do/deploy"
+    echo ""
+    echo "Or set ROLE_ARN in do/config"
+    echo ""
+    echo "The execution role must have permissions for:"
+    echo "  • SageMaker model and transform job management"
+    echo "  • ECR image access"
+    echo "  • S3 read access for input path: ${BATCH_INPUT_PATH}"
+    echo "  • S3 write access for output path: ${BATCH_OUTPUT_PATH}"
+    echo "  • CloudWatch Logs"
+    exit 3
+fi
+
+echo "   Using execution role: ${ROLE_ARN}"
+
+# Validate S3 input path
+if [ -z "${BATCH_INPUT_PATH:-}" ]; then
+    echo "❌ S3 input path not provided"
+    echo ""
+    echo "Set BATCH_INPUT_PATH in do/config or provide via CLI:"
+    echo "  export BATCH_INPUT_PATH=s3://my-bucket/input/"
+    echo "  ./do/deploy"
+    exit 3
+fi
+
+if [[ "${BATCH_INPUT_PATH}" != s3://* ]]; then
+    echo "❌ S3 input path must start with s3://"
+    echo "   Current value: ${BATCH_INPUT_PATH}"
+    echo "   Example: s3://my-bucket/input/"
+    exit 3
+fi
+
+# Validate S3 output path
+if [ -z "${BATCH_OUTPUT_PATH:-}" ]; then
+    echo "❌ S3 output path not provided"
+    echo ""
+    echo "Set BATCH_OUTPUT_PATH in do/config or provide via CLI:"
+    echo "  export BATCH_OUTPUT_PATH=s3://my-bucket/output/"
+    echo "  ./do/deploy"
+    exit 3
+fi
+
+if [[ "${BATCH_OUTPUT_PATH}" != s3://* ]]; then
+    echo "❌ S3 output path must start with s3://"
+    echo "   Current value: ${BATCH_OUTPUT_PATH}"
+    echo "   Example: s3://my-bucket/output/"
+    exit 3
+fi
+
+# ============================================================
+# Bootstrap S3 buckets for batch transform
+# ============================================================
+
+# Extract bucket names from S3 paths
+BATCH_INPUT_BUCKET=$(echo "${BATCH_INPUT_PATH}" | sed 's|s3://||' | cut -d'/' -f1)
+BATCH_OUTPUT_BUCKET=$(echo "${BATCH_OUTPUT_PATH}" | sed 's|s3://||' | cut -d'/' -f1)
+
+<% if (!batchInputPath) { %>
+# Bootstrap default S3 input bucket (check-and-create)
+echo "🔍 Checking if S3 input bucket exists: ${BATCH_INPUT_BUCKET}"
+if ! aws s3api head-bucket --bucket "${BATCH_INPUT_BUCKET}" --region "${AWS_REGION}" 2>/dev/null; then
+    echo "📦 Creating S3 input bucket: ${BATCH_INPUT_BUCKET}"
+    if [ "${AWS_REGION}" = "us-east-1" ]; then
+        if ! aws s3api create-bucket \
+            --bucket "${BATCH_INPUT_BUCKET}" \
+            --region "${AWS_REGION}"; then
+            echo "❌ Failed to create S3 input bucket: ${BATCH_INPUT_BUCKET}"
+            echo ""
+            echo "   Check that:"
+            echo "   • Your IAM credentials have s3:CreateBucket permission"
+            echo "   • The bucket name is not already taken globally"
+            exit 4
+        fi
+    else
+        if ! aws s3api create-bucket \
+            --bucket "${BATCH_INPUT_BUCKET}" \
+            --region "${AWS_REGION}" \
+            --create-bucket-configuration LocationConstraint="${AWS_REGION}"; then
+            echo "❌ Failed to create S3 input bucket: ${BATCH_INPUT_BUCKET}"
+            echo ""
+            echo "   Check that:"
+            echo "   • Your IAM credentials have s3:CreateBucket permission"
+            echo "   • The bucket name is not already taken globally"
+            exit 4
+        fi
+    fi
+    echo "✅ S3 input bucket created: ${BATCH_INPUT_BUCKET}"
+else
+    echo "✅ S3 input bucket exists: ${BATCH_INPUT_BUCKET}"
+fi
+
+# Upload sample input file if the input prefix is empty
+EXISTING_OBJECTS=$(aws s3 ls "${BATCH_INPUT_PATH}" --region "${AWS_REGION}" 2>/dev/null | head -1 || true)
+if [ -z "${EXISTING_OBJECTS}" ]; then
+    echo "📄 Uploading sample input file to ${BATCH_INPUT_PATH}"
+<% if (framework === 'transformers' && (modelServer === 'vllm' || modelServer === 'sglang')) { %>
+    echo '{"model": "<%= modelName %>", "messages": [{"role": "user", "content": "What is machine learning?"}], "max_tokens": 50}' | aws s3 cp - "${BATCH_INPUT_PATH}sample.jsonl" --region "${AWS_REGION}"
+<% } else if (framework === 'transformers') { %>
+    echo '{"inputs": "What is machine learning?", "parameters": {"max_new_tokens": 50}}' | aws s3 cp - "${BATCH_INPUT_PATH}sample.jsonl" --region "${AWS_REGION}"
+<% } else if (framework === 'diffusors') { %>
+    echo '{"prompt": "A white cat", "n": 1, "size": "512x512"}' | aws s3 cp - "${BATCH_INPUT_PATH}sample.jsonl" --region "${AWS_REGION}"
+<% } else { %>
+    echo '{"instances": [[1.0, 2.0, 3.0, 4.0]]}' | aws s3 cp - "${BATCH_INPUT_PATH}sample.jsonl" --region "${AWS_REGION}"
+<% } %>
+    echo "✅ Sample input uploaded: ${BATCH_INPUT_PATH}sample.jsonl"
+    echo "   ⚠️  Replace this with your actual input data before running production jobs"
+fi
+<% } else { %>
+# Custom S3 input path provided — skip bucket creation
+echo "✅ Using custom S3 input path: ${BATCH_INPUT_PATH}"
+<% } %>
+
+<% if (!batchOutputPath) { %>
+# Bootstrap default S3 output bucket (check-and-create, may be same as input)
+if [ "${BATCH_OUTPUT_BUCKET}" != "${BATCH_INPUT_BUCKET}" ]; then
+    echo "🔍 Checking if S3 output bucket exists: ${BATCH_OUTPUT_BUCKET}"
+    if ! aws s3api head-bucket --bucket "${BATCH_OUTPUT_BUCKET}" --region "${AWS_REGION}" 2>/dev/null; then
+        echo "📦 Creating S3 output bucket: ${BATCH_OUTPUT_BUCKET}"
+        if [ "${AWS_REGION}" = "us-east-1" ]; then
+            if ! aws s3api create-bucket \
+                --bucket "${BATCH_OUTPUT_BUCKET}" \
+                --region "${AWS_REGION}"; then
+                echo "❌ Failed to create S3 output bucket: ${BATCH_OUTPUT_BUCKET}"
+                exit 4
+            fi
+        else
+            if ! aws s3api create-bucket \
+                --bucket "${BATCH_OUTPUT_BUCKET}" \
+                --region "${AWS_REGION}" \
+                --create-bucket-configuration LocationConstraint="${AWS_REGION}"; then
+                echo "❌ Failed to create S3 output bucket: ${BATCH_OUTPUT_BUCKET}"
+                exit 4
+            fi
+        fi
+        echo "✅ S3 output bucket created: ${BATCH_OUTPUT_BUCKET}"
+    else
+        echo "✅ S3 output bucket exists: ${BATCH_OUTPUT_BUCKET}"
+    fi
+else
+    echo "✅ S3 output bucket same as input: ${BATCH_OUTPUT_BUCKET}"
+fi
+<% } else { %>
+# Custom S3 output path provided — skip bucket creation
+echo "✅ Using custom S3 output path: ${BATCH_OUTPUT_PATH}"
+<% } %>
+
+# Helper: persist a variable to do/config so other scripts can use it
+_update_config_var() {
+    local var_name="$1" var_value="$2" config_file="${SCRIPT_DIR}/config"
+    if grep -q "^export ${var_name}=" "${config_file}" 2>/dev/null; then
+        sed -i.bak "s|^export ${var_name}=.*|export ${var_name}=\"${var_value}\"|" "${config_file}"
+        rm -f "${config_file}.bak"
+    else
+        echo "" >> "${config_file}"
+        echo "export ${var_name}=\"${var_value}\"" >> "${config_file}"
+    fi
+}
+
+# ============================================================
+# Check for previous transform job still running
+# ============================================================
+if [ "${FORCE_NEW}" != true ] && [ -n "${TRANSFORM_JOB_NAME:-}" ]; then
+    echo "🔍 Checking previous transform job: ${TRANSFORM_JOB_NAME}"
+    PREV_JOB_STATUS=$(aws sagemaker describe-transform-job \
+        --transform-job-name "${TRANSFORM_JOB_NAME}" \
+        --region "${AWS_REGION}" \
+        --query "TransformJobStatus" \
+        --output text 2>/dev/null || echo "")
+
+    case "${PREV_JOB_STATUS}" in
+        InProgress)
+            echo "⚠️  Previous transform job is still running: ${TRANSFORM_JOB_NAME}"
+            echo "   Wait for it to complete, or stop it with:"
+            echo "   aws sagemaker stop-transform-job --transform-job-name ${TRANSFORM_JOB_NAME} --region ${AWS_REGION}"
+            echo ""
+            echo "   Use --force to create a new job anyway."
+            exit 4
+            ;;
+        Completed)
+            echo "✅ Previous transform job completed: ${TRANSFORM_JOB_NAME}"
+            echo "   Creating a new job. Results from the previous job are in:"
+            echo "   ${BATCH_OUTPUT_PATH}"
+            echo ""
+            ;;
+        *)
+            # Failed, Stopped, or not found — proceed with new job
+            ;;
+    esac
+fi
+
+# Generate unique names with timestamp
+TIMESTAMP=$(date +%s)
+MODEL_NAME_SM="${PROJECT_NAME}-batch-model-${TIMESTAMP}"
+TRANSFORM_JOB_NAME="${PROJECT_NAME}-batch-job-${TIMESTAMP}"
+
+_update_config_var "TRANSFORM_JOB_NAME" "${TRANSFORM_JOB_NAME}"
+_update_config_var "SAGEMAKER_MODEL_NAME" "${MODEL_NAME_SM}"
+
+# Step 1: Create SageMaker model
+echo "📦 Creating SageMaker model: ${MODEL_NAME_SM}"
+if ! aws sagemaker create-model \
+    --model-name "${MODEL_NAME_SM}" \
+    --primary-container "{\"Image\":\"${ECR_REPOSITORY}:${IMAGE_TAG}\"}" \
+    --execution-role-arn "${ROLE_ARN}" \
+    --region "${AWS_REGION}"; then
+
+    echo "❌ Failed to create SageMaker model"
+    echo "   Check that:"
+    echo "   • The execution role ARN is valid"
+    echo "   • The ECR image exists and is accessible"
+    echo "   • The IAM role has ecr:GetDownloadUrlForLayer permission"
+    exit 4
+fi
+
+echo "✅ SageMaker model created: ${MODEL_NAME_SM}"
+
+# Record model in manifest (non-blocking)
+MODEL_ARN="arn:aws:sagemaker:${AWS_REGION}:${AWS_ACCOUNT_ID}:model/${MODEL_NAME_SM}"
+./do/manifest add \
+    --type sagemaker-model \
+    --id "${MODEL_ARN}" \
+    --project "${PROJECT_NAME}" \
+    --meta "{\"modelName\":\"${MODEL_NAME_SM}\",\"region\":\"${AWS_REGION}\"}" \
+    2>/dev/null || true
+
+# Step 2: Build transform job JSON
+TRANSFORM_JOB_JSON="{
+    \"TransformJobName\": \"${TRANSFORM_JOB_NAME}\",
+    \"ModelName\": \"${MODEL_NAME_SM}\",
+    \"TransformInput\": {
+        \"DataSource\": {
+            \"S3DataSource\": {
+                \"S3DataType\": \"S3Prefix\",
+                \"S3Uri\": \"${BATCH_INPUT_PATH}\"
+            }
+        },
+        \"ContentType\": \"application/json\",
+        \"SplitType\": \"${BATCH_SPLIT_TYPE}\"
+    },
+    \"TransformOutput\": {
+        \"S3OutputPath\": \"${BATCH_OUTPUT_PATH}\"
+        $([ "${BATCH_JOIN_SOURCE:-None}" = "Input" ] && echo ",\"Accept\": \"application/json\", \"AssembleWith\": \"${BATCH_SPLIT_TYPE}\"")
+    },
+    \"TransformResources\": {
+        \"InstanceType\": \"${INSTANCE_TYPE}\",
+        \"InstanceCount\": ${BATCH_INSTANCE_COUNT}
+    },
+    \"MaxConcurrentTransforms\": ${BATCH_MAX_CONCURRENT_TRANSFORMS:-1},
+    \"MaxPayloadInMB\": ${BATCH_MAX_PAYLOAD_IN_MB:-6},
+    \"BatchStrategy\": \"${BATCH_STRATEGY}\"
+    $([ "${BATCH_JOIN_SOURCE:-None}" = "Input" ] && echo ",\"DataProcessing\": { \"JoinSource\": \"Input\" }")
+}"
+
+# Step 3: Create transform job
+echo "🚀 Creating transform job: ${TRANSFORM_JOB_NAME}"
+if ! aws sagemaker create-transform-job \
+    --cli-input-json "${TRANSFORM_JOB_JSON}" \
+    --region "${AWS_REGION}"; then
+
+    echo "❌ Failed to create transform job"
+    echo "   Check that:"
+    echo "   • The S3 input path exists and is accessible: ${BATCH_INPUT_PATH}"
+    echo "   • The S3 output path is writable: ${BATCH_OUTPUT_PATH}"
+    echo "   • The IAM role has s3:GetObject permission on the input path"
+    echo "   • The IAM role has s3:PutObject permission on the output path"
+    echo "   • The instance type is valid: ${INSTANCE_TYPE}"
+    echo "   • The instance type is available in region: ${AWS_REGION}"
+    echo "   • You have sufficient service quota for the instance type"
+    exit 4
+fi
+
+echo "✅ Transform job created: ${TRANSFORM_JOB_NAME}"
+
+# Record transform job in manifest (non-blocking)
+TRANSFORM_JOB_ARN="arn:aws:sagemaker:${AWS_REGION}:${AWS_ACCOUNT_ID}:transform-job/${TRANSFORM_JOB_NAME}"
+./do/manifest add \
+    --type sagemaker-transform-job \
+    --id "${TRANSFORM_JOB_ARN}" \
+    --project "${PROJECT_NAME}" \
+    --meta "{\"transformJobName\":\"${TRANSFORM_JOB_NAME}\",\"modelName\":\"${MODEL_NAME_SM}\",\"instanceType\":\"${INSTANCE_TYPE}\",\"region\":\"${AWS_REGION}\"}" \
+    2>/dev/null || true
+
+# Step 4: Poll transform job status until completion or failure
+echo "⏳ Waiting for transform job to complete..."
+echo "   This may take several minutes depending on dataset size..."
+echo "   If this times out, check status with:"
+echo "   aws sagemaker describe-transform-job --transform-job-name ${TRANSFORM_JOB_NAME} --region ${AWS_REGION}"
+echo ""
+
+while true; do
+    JOB_STATUS=$(aws sagemaker describe-transform-job \
+        --transform-job-name "${TRANSFORM_JOB_NAME}" \
+        --region "${AWS_REGION}" \
+        --query "TransformJobStatus" \
+        --output text 2>&1) || {
+        # Check if it was a credential expiration
+        if echo "${JOB_STATUS}" | grep -qi "expired\|token"; then
+            echo ""
+            echo "⚠️  Credentials expired, but the transform job is still running."
+            echo "   Refresh your credentials and check status with:"
+            echo "   aws sagemaker describe-transform-job --transform-job-name ${TRANSFORM_JOB_NAME} --region ${AWS_REGION} --query TransformJobStatus"
+            exit 4
+        fi
+        echo "❌ Failed to describe transform job: ${TRANSFORM_JOB_NAME}"
+        echo "   Error: ${JOB_STATUS}"
+        exit 4
+    }
+
+    case "${JOB_STATUS}" in
+        Completed)
+            echo "✅ Transform job completed successfully!"
+            break
+            ;;
+        Failed)
+            FAILURE_REASON=$(aws sagemaker describe-transform-job \
+                --transform-job-name "${TRANSFORM_JOB_NAME}" \
+                --region "${AWS_REGION}" \
+                --query "FailureReason" \
+                --output text 2>/dev/null || echo "Unknown")
+            echo "❌ Transform job failed"
+            echo "   Reason: ${FAILURE_REASON}"
+            echo ""
+            echo "   Check CloudWatch Logs for details:"
+            echo "   https://console.aws.amazon.com/cloudwatch/home?region=${AWS_REGION}#logsV2:log-groups/log-group//aws/sagemaker/TransformJobs"
+            echo ""
+            echo "   Verify that:"
+            echo "   • The S3 input path exists and contains data: ${BATCH_INPUT_PATH}"
+            echo "   • The input data format matches the container's expected format"
+            echo "   • The container's /ping and /invocations endpoints work correctly"
+            exit 4
+            ;;
+        Stopped)
+            echo "⚠️  Transform job was stopped"
+            exit 4
+            ;;
+        InProgress)
+            echo "   $(date +%H:%M:%S) Job status: InProgress..."
+            sleep 30
+            ;;
+        *)
+            echo "   $(date +%H:%M:%S) Job status: ${JOB_STATUS}..."
+            sleep 30
+            ;;
+    esac
+done
+
+echo ""
+echo "📋 Deployment Details:"
+echo "   Transform Job: ${TRANSFORM_JOB_NAME}"
+echo "   Model: ${MODEL_NAME_SM}"
+echo "   Region: ${AWS_REGION}"
+echo "   Instance Type: ${INSTANCE_TYPE}"
+echo "   Instance Count: ${BATCH_INSTANCE_COUNT}"
+echo "   Image: ${ECR_REPOSITORY}:${IMAGE_TAG}"
+echo "   S3 Input: ${BATCH_INPUT_PATH}"
+echo "   S3 Output: ${BATCH_OUTPUT_PATH}"
+echo "   Split Type: ${BATCH_SPLIT_TYPE}"
+echo "   Strategy: ${BATCH_STRATEGY}"
+echo ""
+
+# Download results locally
+LOCAL_OUTPUT_DIR="${SCRIPT_DIR}/../batch-output"
+mkdir -p "${LOCAL_OUTPUT_DIR}"
+echo "📥 Downloading results to ${LOCAL_OUTPUT_DIR}/"
+if aws s3 sync "${BATCH_OUTPUT_PATH}" "${LOCAL_OUTPUT_DIR}/" --region "${AWS_REGION}"; then
+    DOWNLOADED=$(ls -1 "${LOCAL_OUTPUT_DIR}" 2>/dev/null | wc -l | tr -d ' ')
+    echo "✅ Downloaded ${DOWNLOADED} file(s) to ${LOCAL_OUTPUT_DIR}/"
+    echo ""
+
+    # Display first output file preview
+    FIRST_FILE=$(ls -1 "${LOCAL_OUTPUT_DIR}" 2>/dev/null | head -1)
+    if [ -n "${FIRST_FILE}" ]; then
+        echo "📄 Sample output (${FIRST_FILE}):"
+        head -5 "${LOCAL_OUTPUT_DIR}/${FIRST_FILE}"
+        LINES=$(wc -l < "${LOCAL_OUTPUT_DIR}/${FIRST_FILE}" | tr -d ' ')
+        if [ "${LINES}" -gt 5 ]; then
+            echo "   ... (${LINES} total lines)"
+        fi
+    fi
+else
+    echo "⚠️  Could not download output files"
+fi
+
+echo ""
+echo "🧪 Review results:"
+echo "   ./do/test"
+echo ""
+echo "📝 Register this deployment:"
+echo "   ./do/register"
+echo ""
+echo "📋 View logs:"
+echo "   ./do/logs"
+echo ""
+echo "🧹 Clean up when done:"
+echo "   ./do/clean"
+
+<% } %>