@aws/ml-container-creator 0.4.0 → 0.5.0

package/templates/do/ic/default.conf

@@ -0,0 +1,32 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Per-IC configuration: default
+# This file defines the primary inference component for the project.
+# It is sourced by do/lib/inference-component.sh during deployment.
+#
+# After deployment, IC_DEPLOYED_NAME and IC_DEPLOYED_AT will be appended
+# by the deploy script to track the active inference component.
+
+export IC_IMAGE_TAG="<%= projectName %>-latest"
+export IC_GPU_COUNT=<%= (typeof icGpuCount !== 'undefined' && icGpuCount != null) ? icGpuCount : 1 %>
+export IC_COPY_COUNT=1
+export IC_MIN_MEMORY_MB=1024
+export IC_STARTUP_TIMEOUT=900
+<% if (typeof instancePoolSpecs !== 'undefined' && instancePoolSpecs && instancePoolSpecs.length > 1) { %>
+
+# Multi-spec IC configuration (auto-generated from instance pool selections)
+# When the endpoint uses instance pools, the IC uses Specifications (plural)
+# with per-instance-type compute resource requirements.
+export IC_MULTI_SPEC=true
+export IC_SPEC_COUNT=<%= instancePoolSpecs.length %>
+<% instancePoolSpecs.forEach(function(spec, idx) { %>
+export IC_SPEC_<%= idx + 1 %>_INSTANCE_TYPE="<%= spec.instanceType %>"
+export IC_SPEC_<%= idx + 1 %>_GPU_COUNT=<%= spec.gpuCount %>
+export IC_SPEC_<%= idx + 1 %>_MIN_MEMORY_MB=<%= spec.minMemoryMb %>
+<% }); %>
+<% } %>
+
+# Optional overrides:
+# export IC_MODEL_NAME="my-model-v2"
+# export IC_CONTAINER_ENV_EXTRA='"KEY":"value"'

package/templates/do/lib/endpoint-config.sh

@@ -0,0 +1,216 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Shared helper: create SageMaker endpoint configuration.
+# Sourced by do/deploy — expects PROJECT_NAME, AWS_REGION to be set by the caller.
+# One of INSTANCE_TYPE or INSTANCE_POOLS must be set (mutually exclusive).
+# Optional: ROLE_ARN, INFERENCE_AMI_VERSION, CAPACITY_RESERVATION_ARN, ASYNC_INFERENCE_CONFIG,
+#           POOL_TIMEOUT (default: 1200), POOL_INSTANCE_COUNT (default: 1), MODEL_NAME_SM.
+
+# _validate_instance_pools()
+#   Validates that all instance types in INSTANCE_POOLS are compatible:
+#   - All types must share the same accelerator generation (same CUDA/AMI requirements)
+#   - Cannot mix CUDA and Neuron accelerator types
+#   - Unknown instance types produce a warning but do not block deployment
+#
+#   Uses a hardcoded map of instance type prefixes to their generation/AMI compatibility:
+#     cuda-11 (AMI 2-x): g4dn, g5, g5g, p3, p4d, p4de
+#     cuda-12 (AMI 3-x): g6, g6e, p5
+#     cuda-next (AMI 4-x): p6, g7e
+#     neuron: inf1, inf2, trn1
+#
+#   Exits with error if incompatible types are detected.
+_validate_instance_pools() {
+    # Map instance family prefixes to their generation
+    # Format: "family_prefix=generation"
+    local -a GENERATION_MAP=(
+        "ml.g4dn.=cuda-11"
+        "ml.g5.=cuda-11"
+        "ml.g5g.=cuda-11"
+        "ml.p3.=cuda-11"
+        "ml.p4d.=cuda-11"
+        "ml.p4de.=cuda-11"
+        "ml.g6.=cuda-12"
+        "ml.g6e.=cuda-12"
+        "ml.p5.=cuda-12"
+        "ml.p5e.=cuda-12"
+        "ml.p5en.=cuda-12"
+        "ml.p6.=cuda-next"
+        "ml.g7e.=cuda-next"
+        "ml.inf1.=neuron"
+        "ml.inf2.=neuron"
+        "ml.trn1.=neuron"
+    )
+
+    # Extract instance types from INSTANCE_POOLS JSON
+    # INSTANCE_POOLS format: [{"InstanceType":"ml.g6e.48xlarge","Priority":1},...]
+    # Use simple string parsing to extract InstanceType values
+    local pool_types=""
+    pool_types=$(echo "${INSTANCE_POOLS}" | grep -oE '"InstanceType"\s*:\s*"[^"]+"' | sed 's/"InstanceType"\s*:\s*"//;s/"$//' || true)
+
+    if [ -z "${pool_types}" ]; then
+        return 0
+    fi
+
+    local first_generation=""
+    local first_type=""
+    local has_unknown=false
+
+    while IFS= read -r instance_type; do
+        [ -z "${instance_type}" ] && continue
+
+        local generation=""
+        for entry in "${GENERATION_MAP[@]}"; do
+            local prefix="${entry%%=*}"
+            local gen="${entry##*=}"
+            if [[ "${instance_type}" == ${prefix}* ]]; then
+                generation="${gen}"
+                break
+            fi
+        done
+
+        if [ -z "${generation}" ]; then
+            echo "   ⚠️  Unknown instance type in pool: ${instance_type} — skipping validation for this type"
+            has_unknown=true
+            continue
+        fi
+
+        if [ -z "${first_generation}" ]; then
+            first_generation="${generation}"
+            first_type="${instance_type}"
+        elif [ "${generation}" != "${first_generation}" ]; then
+            echo "❌ Cannot mix ${first_type} (${first_generation}) and ${instance_type} (${generation}) in same pool — different CUDA/AMI requirements"
+            echo "   All instance types in a pool must share the same InferenceAmiVersion."
+            echo ""
+            echo "   Generation groupings:"
+            echo "     cuda-11 (AMI 2-x): ml.g4dn.*, ml.g5.*, ml.p3.*, ml.p4d.*"
+            echo "     cuda-12 (AMI 3-x): ml.g6.*, ml.g6e.*, ml.p5.*"
+            echo "     neuron:            ml.inf1.*, ml.inf2.*, ml.trn1.*"
+            echo ""
+            echo "   Fix: use instance types from the same generation in your pool."
+            exit 1
+        fi
+    done <<< "${pool_types}"
+}
+
+# create_endpoint_config()
+#   Builds a ProductionVariant JSON and calls `aws sagemaker create-endpoint-config`.
+#   Sets the global ENDPOINT_CONFIG_NAME variable for downstream use.
+#
+#   Behavior:
+#     - INSTANCE_POOLS set: uses InstancePools array, RoutingConfig, VariantInstanceProvisionTimeoutInSeconds
+#       Omits InstanceType entirely (mutually exclusive with pools)
+#     - INSTANCE_POOLS not set: uses single INSTANCE_TYPE (standard path)
+#     - INFERENCE_AMI_VERSION: appended to variant when set
+#     - CAPACITY_RESERVATION_ARN: appended to variant when set (only for single instance type path)
+#     - ASYNC_INFERENCE_CONFIG: passes --async-inference-config when set
+#     - ROLE_ARN + no MODEL_NAME_SM: passes --execution-role-arn (IC-based real-time flow)
+#     - MODEL_NAME_SM set: omits --execution-role-arn (model-based async flow)
+create_endpoint_config() {
+    # Mutual exclusivity: capacity reservations and instance pools cannot be used together.
+    # Capacity reservations guarantee a specific instance type, while pools are for fallback
+    # across multiple types. If both are set, prefer the reservation.
+    if [ -n "${INSTANCE_POOLS:-}" ] && [ -n "${CAPACITY_RESERVATION_ARN:-}" ]; then
+        echo "⚠️  Capacity reservations and instance pools are mutually exclusive. Using capacity reservation."
+        unset INSTANCE_POOLS
+    fi
+
+    local timestamp
+    timestamp=$(date +%s)
+    ENDPOINT_CONFIG_NAME="${PROJECT_NAME}-epc-${timestamp}"
+
+    local variant_json
+
+    if [ -n "${INSTANCE_POOLS:-}" ]; then
+        # Validate pool compatibility before proceeding
+        _validate_instance_pools
+
+        # Instance pools path: heterogeneous instance types with priority-based fallback
+        echo "   Instance pools: enabled"
+
+        # Transform ModelName → ModelNameOverride for the SageMaker API.
+        # INSTANCE_POOLS config uses "ModelName" for readability; the API expects "ModelNameOverride"
+        # as a sibling of InstanceType and Priority within each pool entry.
+        local pools_json="${INSTANCE_POOLS}"
+        if echo "${pools_json}" | grep -q '"ModelName"'; then
+            pools_json=$(echo "${pools_json}" | sed 's/"ModelName"/"ModelNameOverride"/g')
+            echo "   ModelNameOverride: per-pool model names detected"
+        fi
+
+        variant_json="[{\"VariantName\":\"AllTraffic\""
+        variant_json="${variant_json},\"InstancePools\":${pools_json}"
+        variant_json="${variant_json},\"InitialInstanceCount\":${POOL_INSTANCE_COUNT:-1}"
+        variant_json="${variant_json},\"VariantInstanceProvisionTimeoutInSeconds\":${POOL_TIMEOUT:-1200}"
+        variant_json="${variant_json},\"RoutingConfig\":{\"RoutingStrategy\":\"LEAST_OUTSTANDING_REQUESTS\"}"
+
+        # Optional: AMI version
+        if [ -n "${INFERENCE_AMI_VERSION:-}" ]; then
+            variant_json="${variant_json},\"InferenceAmiVersion\":\"${INFERENCE_AMI_VERSION}\""
+            echo "   AMI version: ${INFERENCE_AMI_VERSION}"
+        fi
+
+        variant_json="${variant_json}}]"
+    else
+        # Standard path: single instance type
+        variant_json="[{\"VariantName\":\"AllTraffic\",\"InstanceType\":\"${INSTANCE_TYPE}\",\"InitialInstanceCount\":1"
+
+        # Optional: AMI version
+        if [ -n "${INFERENCE_AMI_VERSION:-}" ]; then
+            variant_json="${variant_json},\"InferenceAmiVersion\":\"${INFERENCE_AMI_VERSION}\""
+            echo "   AMI version: ${INFERENCE_AMI_VERSION}"
+        fi
+
+        # Optional: capacity reservation
+        if [ -n "${CAPACITY_RESERVATION_ARN:-}" ]; then
+            variant_json="${variant_json},\"CapacityReservationConfig\":{\"CapacityReservationPreference\":\"capacity-reservations-only\",\"MlReservationArn\":\"${CAPACITY_RESERVATION_ARN}\"}"
+            echo "   ⚠️  Capacity reservation (experimental): ${CAPACITY_RESERVATION_ARN}"
+        fi
+
+        variant_json="${variant_json}}]"
+    fi
+
+    # Build the AWS CLI command arguments
+    local -a cmd_args=(
+        aws sagemaker create-endpoint-config
+        --endpoint-config-name "${ENDPOINT_CONFIG_NAME}"
+    )
+
+    # Include --execution-role-arn for IC-based flow (real-time).
+    # Omit for model-based flow (async) where MODEL_NAME_SM is set.
+    if [ -n "${ROLE_ARN:-}" ] && [ -z "${MODEL_NAME_SM:-}" ]; then
+        cmd_args+=(--execution-role-arn "${ROLE_ARN}")
+    fi
+
+    cmd_args+=(--production-variants "${variant_json}")
+
+    # Optional: async inference config
+    if [ -n "${ASYNC_INFERENCE_CONFIG:-}" ]; then
+        cmd_args+=(--async-inference-config "${ASYNC_INFERENCE_CONFIG}")
+    fi
+
+    cmd_args+=(--region "${AWS_REGION}")
+
+    echo "⚙️  Creating endpoint configuration: ${ENDPOINT_CONFIG_NAME}"
+    if ! "${cmd_args[@]}"; then
+        echo "❌ Failed to create endpoint configuration"
+        echo "   Check that:"
+        if [ -n "${ROLE_ARN:-}" ] && [ -z "${MODEL_NAME_SM:-}" ]; then
+            echo "   • The execution role ARN is valid"
+        fi
+        if [ -n "${INSTANCE_POOLS:-}" ]; then
+            echo "   • The instance pool types are valid and available in region: ${AWS_REGION}"
+            echo "   • You have sufficient service quota for the pool instance types"
+        else
+            echo "   • The instance type is valid: ${INSTANCE_TYPE}"
+            echo "   • The instance type is available in region: ${AWS_REGION}"
+            echo "   • You have sufficient service quota for the instance type"
+        fi
+        if [ -n "${ASYNC_INFERENCE_CONFIG:-}" ]; then
+            echo "   • The async inference config is valid JSON"
+            echo "   • The S3 output path and SNS topics are accessible"
+        fi
+        exit 4
+    fi
+
+    echo "✅ Endpoint configuration created: ${ENDPOINT_CONFIG_NAME}"
+}

package/templates/do/lib/inference-component.sh

@@ -0,0 +1,167 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Shared helper: create SageMaker inference components.
+# Sourced by do/deploy — expects the following to be set by the caller:
+#   PROJECT_NAME, ENDPOINT_NAME, ECR_REPOSITORY, AWS_REGION, CONTAINER_ENV_JSON
+# Also expects _update_config_var() to be available (from wait.sh).
+
+# create_inference_component <ic_config_file>
+#   Creates an inference component from a per-IC config file.
+#
+#   The config file is sourced and should export:
+#     IC_IMAGE_TAG       — container image tag (default: ${PROJECT_NAME}-latest)
+#     IC_GPU_COUNT       — number of accelerator devices (default: 1)
+#     IC_COPY_COUNT      — number of IC copies (default: 1)
+#     IC_MIN_MEMORY_MB   — minimum memory in MB (default: 1024)
+#     IC_STARTUP_TIMEOUT — container startup health check timeout in seconds (default: 900)
+#     IC_CONTAINER_ENV_EXTRA — optional extra env vars in "KEY":"value" format
+#
+#   Multi-spec support (for heterogeneous instance pools):
+#     IC_MULTI_SPEC      — set to "true" to use Specifications (plural) array
+#     IC_SPEC_COUNT      — number of spec entries (e.g., 2)
+#     IC_SPEC_N_INSTANCE_TYPE — instance type for spec entry N
+#     IC_SPEC_N_GPU_COUNT     — GPU count for spec entry N
+#     IC_SPEC_N_MIN_MEMORY_MB — minimum memory for spec entry N
+#
+#   Sets IC_DEPLOYED_NAME in the caller's scope (for use by wait_ic).
+#   Persists IC_DEPLOYED_NAME and IC_DEPLOYED_AT back to the IC config file.
+#   Echoes the IC name as return value.
+create_inference_component() {
+    local ic_conf="$1"
+
+    if [ ! -f "${ic_conf}" ]; then
+        echo "❌ IC config file not found: ${ic_conf}"
+        exit 4
+    fi
+
+    # Source the IC config to get per-IC settings
+    source "${ic_conf}"
+
+    local ic_timestamp
+    ic_timestamp=$(date +%s)
+    local ic_basename
+    ic_basename=$(basename "${ic_conf}" .conf)
+    local ic_name="${PROJECT_NAME}-${ic_basename}-${ic_timestamp}"
+
+    # Build container spec JSON
+    local container_spec="{\"Image\":\"${ECR_REPOSITORY}:${IC_IMAGE_TAG:-${PROJECT_NAME}-latest}\""
+    if [ -n "${CONTAINER_ENV_JSON}${IC_CONTAINER_ENV_EXTRA:-}" ]; then
+        local env_json="${CONTAINER_ENV_JSON}"
+        [ -n "${IC_CONTAINER_ENV_EXTRA:-}" ] && env_json="${env_json:+${env_json},}${IC_CONTAINER_ENV_EXTRA}"
+        container_spec="${container_spec},\"Environment\":{${env_json}}"
+    fi
+    container_spec="${container_spec}}"
+
+    # Build specification JSON — multi-spec (Specifications array) or single (Specification object)
+    local spec_json
+    if [ "${IC_MULTI_SPEC:-false}" = "true" ] && [ "${IC_SPEC_COUNT:-0}" -gt 0 ]; then
+        # Multi-spec: build Specifications array with per-instance-type compute resources
+        spec_json="{\"Specifications\":["
+        local i=1
+        while [ "${i}" -le "${IC_SPEC_COUNT}" ]; do
+            local spec_instance_type_var="IC_SPEC_${i}_INSTANCE_TYPE"
+            local spec_gpu_count_var="IC_SPEC_${i}_GPU_COUNT"
+            local spec_min_memory_var="IC_SPEC_${i}_MIN_MEMORY_MB"
+
+            local spec_instance_type="${!spec_instance_type_var}"
+            local spec_gpu_count="${!spec_gpu_count_var:-1}"
+            local spec_min_memory="${!spec_min_memory_var:-1024}"
+
+            if [ "${i}" -gt 1 ]; then
+                spec_json="${spec_json},"
+            fi
+            spec_json="${spec_json}{\"Container\":${container_spec},\"StartupParameters\":{\"ContainerStartupHealthCheckTimeoutInSeconds\":${IC_STARTUP_TIMEOUT:-900}},\"ComputeResourceRequirements\":{\"NumberOfAcceleratorDevicesRequired\":${spec_gpu_count},\"MinMemoryRequiredInMb\":${spec_min_memory}}}"
+
+            i=$((i + 1))
+        done
+        spec_json="${spec_json}]}"
+    else
+        # Single spec: standard Specification object (existing behavior)
+        spec_json="{\"Container\":${container_spec},\"StartupParameters\":{\"ContainerStartupHealthCheckTimeoutInSeconds\":${IC_STARTUP_TIMEOUT:-900}},\"ComputeResourceRequirements\":{\"NumberOfAcceleratorDevicesRequired\":${IC_GPU_COUNT:-1},\"MinMemoryRequiredInMb\":${IC_MIN_MEMORY_MB:-1024}}}"
+    fi
+
+    echo "📦 Creating inference component: ${ic_name}"
+    if ! aws sagemaker create-inference-component \
+        --inference-component-name "${ic_name}" \
+        --endpoint-name "${ENDPOINT_NAME}" \
+        --variant-name "AllTraffic" \
+        --specification "${spec_json}" \
+        --runtime-config "{\"CopyCount\": ${IC_COPY_COUNT:-1}}" \
+        --region "${AWS_REGION}"; then
+
+        echo "❌ Failed to create inference component: ${ic_name}"
+        echo "   Check that:"
+        echo "   • The endpoint is InService: ${ENDPOINT_NAME}"
+        echo "   • The container image exists: ${ECR_REPOSITORY}:${IC_IMAGE_TAG:-${PROJECT_NAME}-latest}"
+        echo "   • GPU count (${IC_GPU_COUNT:-1}) does not exceed instance capacity"
+        echo "   • You have sufficient permissions for sagemaker:CreateInferenceComponent"
+        exit 4
+    fi
+
+    # Persist deployed name and timestamp back to IC config
+    IC_DEPLOYED_NAME="${ic_name}"
+    IC_DEPLOYED_AT="${ic_timestamp}"
+    _update_config_var "IC_DEPLOYED_NAME" "${ic_name}" "${ic_conf}"
+    _update_config_var "IC_DEPLOYED_AT" "${ic_timestamp}" "${ic_conf}"
+
+    echo "✅ Inference component created: ${ic_name}"
+    echo "${ic_name}"
+}
+
+# create_inference_component_legacy()
+#   Backward-compatible IC creation for projects without do/ic/ directory.
+#   Reads IC_GPU_COUNT from do/config (already sourced) and IMAGE_TAG from caller scope.
+#   Uses the same endpoint and container env as the multi-IC path.
+#
+#   Sets IC_DEPLOYED_NAME in the caller's scope (for use by wait_ic).
+#   Persists INFERENCE_COMPONENT_NAME to do/config.
+create_inference_component_legacy() {
+    local ic_timestamp
+    ic_timestamp=$(date +%s)
+    local ic_name="${PROJECT_NAME}-ic-${ic_timestamp}"
+
+    # Build container spec JSON (uses IMAGE_TAG from caller scope)
+    local container_spec="{\"Image\":\"${ECR_REPOSITORY}:${IMAGE_TAG}\""
+    if [ -n "${CONTAINER_ENV_JSON}" ]; then
+        container_spec="${container_spec},\"Environment\":{${CONTAINER_ENV_JSON}}"
+    fi
+    container_spec="${container_spec}}"
+
+    echo "📦 Creating inference component: ${ic_name}"
+    if ! aws sagemaker create-inference-component \
+        --inference-component-name "${ic_name}" \
+        --endpoint-name "${ENDPOINT_NAME}" \
+        --variant-name "AllTraffic" \
+        --specification "{
+            \"Container\": ${container_spec},
+            \"StartupParameters\": {
+                \"ContainerStartupHealthCheckTimeoutInSeconds\": 900
+            },
+            \"ComputeResourceRequirements\": {
+                \"NumberOfAcceleratorDevicesRequired\": ${IC_GPU_COUNT:-1},
+                \"MinMemoryRequiredInMb\": 1024
+            }
+        }" \
+        --runtime-config "{\"CopyCount\": 1}" \
+        --region "${AWS_REGION}"; then
+
+        echo "❌ Failed to create inference component: ${ic_name}"
+        echo "   Check that:"
+        echo "   • The endpoint is InService: ${ENDPOINT_NAME}"
+        echo "   • The container image exists: ${ECR_REPOSITORY}:${IMAGE_TAG}"
+        echo "   • GPU count (${IC_GPU_COUNT:-1}) does not exceed instance capacity"
+        echo "   • You have sufficient permissions for sagemaker:CreateInferenceComponent"
+        exit 4
+    fi
+
+    # Set in caller's scope for wait_ic
+    IC_DEPLOYED_NAME="${ic_name}"
+    IC_DEPLOYED_AT="${ic_timestamp}"
+
+    # Persist to do/config for legacy compatibility
+    _update_config_var "INFERENCE_COMPONENT_NAME" "${ic_name}"
+    _update_config_var "IC_DEPLOYED_AT" "${ic_timestamp}"
+
+    echo "✅ Inference component created: ${ic_name}"
+}

package/templates/do/lib/secrets.sh

@@ -0,0 +1,44 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Shared helper: resolve container secrets from Secrets Manager or direct values.
+# Sourced by do/deploy — expects AWS_REGION to be set by the caller.
+
+# resolve_secrets()
+#   Resolves HF_TOKEN and NGC_API_KEY from either:
+#     - AWS Secrets Manager (when *_ARN variables are set)
+#     - Direct values (when the plain variables are set)
+#   Sets the global CONTAINER_ENV_JSON variable with comma-separated "KEY":"value" pairs.
+resolve_secrets() {
+    CONTAINER_ENV_JSON=""
+
+    if [ -n "${HF_TOKEN_ARN:-}" ]; then
+        echo "🔐 Resolving HuggingFace token from Secrets Manager..."
+        RESOLVED_HF_TOKEN=$(aws secretsmanager get-secret-value --secret-id "${HF_TOKEN_ARN}" --query SecretString --output text --region "${AWS_REGION}") || {
+            echo "❌ Failed to resolve HuggingFace token from Secrets Manager"
+            exit 3
+        }
+        CONTAINER_ENV_JSON="\"HF_TOKEN\":\"${RESOLVED_HF_TOKEN}\""
+    elif [ -n "${HF_TOKEN:-}" ]; then
+        CONTAINER_ENV_JSON="\"HF_TOKEN\":\"${HF_TOKEN}\""
+    fi
+
+    if [ -n "${NGC_API_KEY_ARN:-}" ]; then
+        echo "🔐 Resolving NGC API key from Secrets Manager..."
+        RESOLVED_NGC_KEY=$(aws secretsmanager get-secret-value --secret-id "${NGC_API_KEY_ARN}" --query SecretString --output text --region "${AWS_REGION}") || {
+            echo "❌ Failed to resolve NGC API key from Secrets Manager"
+            exit 3
+        }
+        if [ -n "${CONTAINER_ENV_JSON}" ]; then
+            CONTAINER_ENV_JSON="${CONTAINER_ENV_JSON},\"NGC_API_KEY\":\"${RESOLVED_NGC_KEY}\""
+        else
+            CONTAINER_ENV_JSON="\"NGC_API_KEY\":\"${RESOLVED_NGC_KEY}\""
+        fi
+    elif [ -n "${NGC_API_KEY:-}" ]; then
+        if [ -n "${CONTAINER_ENV_JSON}" ]; then
+            CONTAINER_ENV_JSON="${CONTAINER_ENV_JSON},\"NGC_API_KEY\":\"${NGC_API_KEY}\""
+        else
+            CONTAINER_ENV_JSON="\"NGC_API_KEY\":\"${NGC_API_KEY}\""
+        fi
+    fi
+}

package/templates/do/lib/wait.sh

@@ -0,0 +1,131 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Shared helper: wait/polling functions and config persistence utilities.
+# Sourced by do/deploy — expects AWS_REGION and SCRIPT_DIR to be set by the caller.
+
+# _update_config_var <var_name> <var_value> [config_file]
+#   Persist a variable to a config file so other scripts can use it.
+#   If the variable already exists, update it in place; otherwise append.
+#   Defaults to ${SCRIPT_DIR}/config if no config_file is specified.
+_update_config_var() {
+    local var_name="$1" var_value="$2" config_file="${3:-${SCRIPT_DIR}/config}"
+    if grep -q "^export ${var_name}=" "${config_file}" 2>/dev/null; then
+        sed -i.bak "s|^export ${var_name}=.*|export ${var_name}=\"${var_value}\"|" "${config_file}"
+        rm -f "${config_file}.bak"
+    else
+        echo "" >> "${config_file}"
+        echo "export ${var_name}=\"${var_value}\"" >> "${config_file}"
+    fi
+}
+
+# _get_endpoint_status <endpoint_name>
+#   Query a SageMaker endpoint status. Returns empty string if not found.
+_get_endpoint_status() {
+    aws sagemaker describe-endpoint \
+        --endpoint-name "$1" \
+        --region "${AWS_REGION}" \
+        --query EndpointStatus \
+        --output text 2>/dev/null || echo ""
+}
+
+# _get_ic_status <inference_component_name>
+#   Query a SageMaker inference component status. Returns empty string if not found.
+_get_ic_status() {
+    aws sagemaker describe-inference-component \
+        --inference-component-name "$1" \
+        --region "${AWS_REGION}" \
+        --query InferenceComponentStatus \
+        --output text 2>/dev/null || echo ""
+}
+
+# _find_active_ic_on_endpoint <endpoint_name>
+#   Find an InService inference component on an endpoint.
+#   Returns the first match or empty string.
+_find_active_ic_on_endpoint() {
+    aws sagemaker list-inference-components \
+        --endpoint-name "$1" \
+        --status-equals InService \
+        --region "${AWS_REGION}" \
+        --query 'InferenceComponents[0].InferenceComponentName' \
+        --output text 2>/dev/null || echo ""
+}
+
+# wait_endpoint <endpoint_name>
+#   Wait for a SageMaker endpoint to reach InService status.
+#   Detects credential expiry vs actual failure and exits with code 4 on error.
+wait_endpoint() {
+    local endpoint_name="$1"
+
+    if ! aws sagemaker wait endpoint-in-service \
+        --endpoint-name "${endpoint_name}" \
+        --region "${AWS_REGION}"; then
+
+        # Check if it was a credential expiration vs actual failure
+        local ep_check
+        ep_check=$(_get_endpoint_status "${endpoint_name}" 2>/dev/null)
+        if [ "${ep_check}" = "Creating" ]; then
+            echo ""
+            echo "⚠️  Wait interrupted (credentials may have expired), but endpoint is still creating."
+            echo "   Refresh your credentials and re-run ./do/deploy to resume."
+            exit 4
+        fi
+
+        echo "❌ Endpoint failed to reach InService status"
+        echo "   Check CloudWatch Logs for details:"
+        echo "   https://console.aws.amazon.com/cloudwatch/home?region=${AWS_REGION}#logsV2:log-groups/log-group//aws/sagemaker/Endpoints/${endpoint_name}"
+        exit 4
+    fi
+}
+
+# wait_ic <ic_name> [timeout]
+#   Poll an inference component until it reaches InService or fails.
+#   Default timeout is 1800 seconds (30 minutes).
+#   Reports status every 30 seconds. Detects credential expiry.
+#   Exits with code 4 on failure or timeout.
+wait_ic() {
+    local ic_name="$1"
+    local timeout="${2:-1800}"
+    local wait_start
+    wait_start=$(date +%s)
+
+    while true; do
+        local ic_status
+        ic_status=$(_get_ic_status "${ic_name}" 2>/dev/null)
+
+        case "${ic_status}" in
+            InService)
+                break
+                ;;
+            Failed)
+                echo "❌ Inference component failed to reach InService status"
+                echo "   Check CloudWatch Logs for details:"
+                echo "   https://console.aws.amazon.com/cloudwatch/home?region=${AWS_REGION}#logsV2:log-groups/log-group//aws/sagemaker/Endpoints/${ENDPOINT_NAME:-unknown}"
+                echo ""
+                echo "   Debug:"
+                echo "   aws sagemaker describe-inference-component --inference-component-name ${ic_name} --region ${AWS_REGION}"
+                exit 4
+                ;;
+            Creating)
+                local elapsed=$(( $(date +%s) - wait_start ))
+                if [ "${elapsed}" -ge "${timeout}" ]; then
+                    echo ""
+                    echo "⚠️  Inference component still creating after ${timeout}s."
+                    echo "   Re-run ./do/deploy to resume waiting."
+                    exit 4
+                fi
+                echo "   $(date +%H:%M:%S) Status: Creating (${elapsed}s elapsed)..."
+                sleep 30
+                ;;
+            "")
+                echo "⚠️  Could not determine inference component status (credentials may have expired)."
+                echo "   Re-run ./do/deploy to resume."
+                exit 4
+                ;;
+            *)
+                echo "   $(date +%H:%M:%S) Status: ${ic_status}..."
+                sleep 30
+                ;;
+        esac
+    done
+}