@aws/ml-container-creator 0.13.3 → 0.13.5

package/servers/lib/catalogs/instances.json

@@ -228,33 +228,6 @@
             "gpuMemoryGb": 24,
             "gpuType": "NVIDIA A10G",
             "costTier": "medium"
-        },
-        "ml.p6-b200.48xlarge": {
-            "category": "gpu",
-            "gpus": 8,
-            "vcpus": 192,
-            "memGb": 1536,
-            "accelerator": "8x B200 1440GB",
-            "cudaVersions": [
-                "12.4",
-                "12.6"
-            ],
-            "tags": [
-                "gpu",
-                "multi-gpu",
-                "b200",
-                "cuda-12",
-                "high-performance"
-            ],
-            "family": "p6",
-            "acceleratorType": "cuda",
-            "hardware": "NVIDIA B200",
-            "gpuArchitecture": "Blackwell",
-            "defaultCudaVersion": "12.6",
-            "notes": "8x NVIDIA B200 GPUs (1440GB total). Next-gen Blackwell architecture",
-            "gpuMemoryGb": 180,
-            "gpuType": "NVIDIA B200",
-            "costTier": "high"
         }
     },
     "recommendations": {

package/src/app.js

@@ -400,6 +400,8 @@ export async function writeProject(templateDir, destDir, answers, registryConfig
         ignorePatterns.push('**/do/adapters/**');
         ignorePatterns.push('**/do/tune');
         ignorePatterns.push('**/do/.tune_helper.py');
+        ignorePatterns.push('**/do/.stage_helper.py');
+        ignorePatterns.push('**/do/.adapter_helper.py');
         ignorePatterns.push('**/do/train');
         ignorePatterns.push('**/do/.train_build_request.py');
         ignorePatterns.push('**/do/.train_status_parser.py');

package/src/lib/bootstrap-command-handler.js

@@ -170,7 +170,7 @@ export default class BootstrapCommandHandler {
         console.log('\n🚀 Bootstrap — Shared AWS Infrastructure Setup\n');
 
         // Verify AWS CLI v2 is installed
-        if (!this.provisioners._verifyCliV2()) {
+        if (!this._verifyCliV2()) {
             return;
         }
 
@@ -261,7 +261,7 @@ export default class BootstrapCommandHandler {
                 }
 
                 profileData.roleArn = stackOutputs.RoleArn;
-                profileData.ecrRepositoryName = stackOutputs.EcrRepositoryName;
+                profileData.ecrRepositoryName = stackOutputs.EcrRepositoryName || 'ml-container-creator';
                 profileData.stackName = stackName;
                 profileData.sharedInfraFrom = otherStack;  // Track that this profile reuses another's stack
                 if (stackOutputs.AsyncS3BucketName) profileData.asyncS3Bucket = stackOutputs.AsyncS3BucketName;
@@ -459,39 +459,49 @@ export default class BootstrapCommandHandler {
 
                     // --no-rollback prevents rollback on AlreadyExists errors for IAM roles
                     // that may pre-exist from a prior deployment or another region.
-                    // Check if benchmark bucket already exists (from a prior torn-down stack with RETAIN policy)
-                    let importBucketCtx = '';
+                    // Check if benchmark results bucket already exists.
+                    // If it does, skip CDK deploy for benchmark infra — just update the profile.
+                    let benchmarkBucketExists = false;
                     if (options.benchmarkInfra) {
+                        const resultsBucketName = `mlcc-benchmark-results-${profileData.accountId}-${profileData.awsRegion}`;
                         try {
                             execSync(
-                                `aws s3api head-bucket --bucket mlcc-benchmark-results-${profileData.accountId}-${profileData.awsRegion}${profileData.awsProfile ? ` --profile ${profileData.awsProfile}` : ''} --region ${profileData.awsRegion}`,
+                                `aws s3api head-bucket --bucket ${resultsBucketName}${profileData.awsProfile ? ` --profile ${profileData.awsProfile}` : ''} --region ${profileData.awsRegion}`,
                                 { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }
                             );
-                            importBucketCtx = ' -c importExistingBenchmarkBucket=true';
-                            console.log('  ℹ️  Benchmark results bucket already exists — importing into stack');
+                            benchmarkBucketExists = true;
+                            console.log(`  ✅ Benchmark results bucket already exists: ${resultsBucketName}`);
+                            console.log('     Skipping CDK deploy for benchmark infra — updating profile only.');
+                            profileData.benchmarkInfraProvisioned = true;
+                            profileData.ciGlueDatabase = profileData.ciGlueDatabase || 'mlcc_ci';
+                            profileData.ciBenchmarkResultsBucket = resultsBucketName;
                         } catch {
                             // Bucket doesn't exist — will be created fresh
                         }
                     }
-                    const cdkDeployCmd = options.benchmarkInfra
-                        ? `npx cdk deploy MlccCiHarnessStack --require-approval never --no-rollback --parameters MlccCiHarnessStack:CreateBenchmarkInfra=true${importBucketCtx}`
-                        : 'npx cdk deploy MlccCiHarnessStack --require-approval never --no-rollback';
-                    execSync(
-                        cdkDeployCmd,
-                        {
-                            cwd: ciHarnessDir,
-                            encoding: 'utf8',
-                            stdio: 'inherit',
-                            env: {
-                                ...process.env,
-                                AWS_REGION: profileData.awsRegion,
-                                CDK_DEFAULT_REGION: profileData.awsRegion,
-                                CDK_DEFAULT_ACCOUNT: profileData.accountId,
-                                AWS_PROFILE: profileData.awsProfile
+
+                    // Only run CDK deploy if we actually need to create infrastructure
+                    if (!benchmarkBucketExists || !options.benchmarkInfra) {
+                        const cdkDeployCmd = options.benchmarkInfra
+                            ? 'npx cdk deploy MlccCiHarnessStack --require-approval never --no-rollback --parameters MlccCiHarnessStack:CreateBenchmarkInfra=true'
+                            : 'npx cdk deploy MlccCiHarnessStack --require-approval never --no-rollback';
+                        execSync(
+                            cdkDeployCmd,
+                            {
+                                cwd: ciHarnessDir,
+                                encoding: 'utf8',
+                                stdio: 'inherit',
+                                env: {
+                                    ...process.env,
+                                    AWS_REGION: profileData.awsRegion,
+                                    CDK_DEFAULT_REGION: profileData.awsRegion,
+                                    CDK_DEFAULT_ACCOUNT: profileData.accountId,
+                                    AWS_PROFILE: profileData.awsProfile
+                                }
                             }
-                        }
-                    );
-                    console.log('  ✅ CI harness stack deployed');
+                        );
+                        console.log('  ✅ CI harness stack deployed');
+                    }
 
                     profileData.ciInfraProvisioned = true;
                     profileData.ciTableName = 'mlcc-ci-table';

package/src/lib/generated/cli-options.js

@@ -1,6 +1,6 @@
 // AUTO-GENERATED by scripts/codegen-cli.js — DO NOT EDIT
 // Source: config/parameter-schema-v2.json
-// Generated: 2026-06-12T22:03:00.429Z
+// Generated: 2026-06-15T20:16:03.840Z
 
 /**
  * CLI option definitions derived from parameter-schema-v2.json.

package/src/lib/generated/parameter-matrix.js

@@ -1,6 +1,6 @@
 // AUTO-GENERATED by scripts/codegen-parameter-matrix.js — DO NOT EDIT
 // Source: config/parameter-schema-v2.json
-// Generated: 2026-06-12T22:03:00.552Z
+// Generated: 2026-06-15T20:16:03.952Z
 
 /**
  * Parameter matrix defining how each parameter is loaded from various sources.

package/src/lib/generated/validation-rules.js

@@ -1,6 +1,6 @@
 // AUTO-GENERATED by scripts/codegen-validator.js — DO NOT EDIT
 // Source: config/parameter-schema-v2.json
-// Generated: 2026-06-12T22:03:00.468Z
+// Generated: 2026-06-15T20:16:03.877Z
 
 /**
  * Validation rules derived from parameter-schema-v2.json.

package/src/lib/prompt-runner.js

@@ -18,8 +18,6 @@ import {
     modelLoadStrategyPrompts,
     modelProfilePrompts,
     modulePrompts,
-    loraPrompts,
-    benchmarkPrompts,
     infraRegionAndTargetPrompts,
     infraExistingEndpointPrompts,
     infraInstancePrompts,
@@ -521,38 +519,23 @@ export default class PromptRunner {
         const ngcApiKeyAnswers = { ngcApiKey: secretAnswers.ngcApiKey, ngcTokenArn: secretAnswers.ngcTokenArn };
 
         // Module selection
-        const moduleAnswers = await this._runPhase(modulePrompts, { ...frameworkAnswers, ...engineAnswers }, explicitConfig, existingConfig);
-        
-        // Ensure transformers, diffusors, and ineligible Triton backends don't get sample model
-        if (frameworkAnswers.architecture === 'transformers' ||
-            frameworkAnswers.architecture === 'diffusors' ||
-            (frameworkAnswers.architecture === 'triton' && 
-             !this._tritonBackends[frameworkAnswers.backend]?.supportsSampleModel)) {
+        // Only ask about sample model for non-transformers/diffusors (Triton etc.)
+        const moduleAnswers = {};
+        if (frameworkAnswers.architecture !== 'transformers' &&
+            frameworkAnswers.architecture !== 'diffusors') {
+            const sampleModelAnswers = await this._runPhase(
+                modulePrompts.filter(p => p.name === 'includeSampleModel'),
+                { ...frameworkAnswers, ...engineAnswers }, explicitConfig, existingConfig
+            );
+            Object.assign(moduleAnswers, sampleModelAnswers);
+        } else {
             moduleAnswers.includeSampleModel = false;
         }
 
-        // Benchmark prompts — derive includeBenchmark from testTypes selection or CLI flag
-        // Requirements: 1.1, 1.2
-        let benchmarkAnswers = {};
-        if (frameworkAnswers.architecture === 'transformers' || frameworkAnswers.architecture === 'diffusors') {
-            const testTypes = moduleAnswers.testTypes || [];
-            const includeBenchmark = testTypes.includes('sagemaker-ai-automated-benchmarking') ||
-                explicitConfig.includeBenchmark === true ||
-                explicitConfig.includeBenchmark === 'true';
-            benchmarkAnswers.includeBenchmark = includeBenchmark;
-            if (includeBenchmark) {
-                const subAnswers = await this._runPhase(benchmarkPrompts, { ...frameworkAnswers, ...moduleAnswers, includeBenchmark }, explicitConfig, existingConfig);
-                benchmarkAnswers = { ...benchmarkAnswers, ...subAnswers };
-            }
-        }
-
-        // LoRA adapter prompts — only for transformers with vllm/sglang/djl-lmi
-        // Requirements: 1.1, 1.2, 1.4
-        let loraAnswers = {};
-        const loraSubAnswers = await this._runPhase(loraPrompts, { ...frameworkAnswers, ...engineAnswers }, explicitConfig, existingConfig);
-        if (loraSubAnswers.enableLora !== undefined) {
-            loraAnswers = loraSubAnswers;
-        }
+        // Test types, benchmark, and LoRA are always-on (BL-122)
+        moduleAnswers.testTypes = ['hosted-model-endpoint', 'sagemaker-ai-automated-benchmarking'];
+        const benchmarkAnswers = { includeBenchmark: true };
+        const loraAnswers = { enableLora: true };
 
         // Validate instance type against framework requirements (now that framework version is known)
         const finalInstanceType = infraAnswers.customInstanceType || infraAnswers.instanceType;

package/templates/IAM_PERMISSIONS.md

@@ -10,14 +10,47 @@ This project uses three sets of IAM permissions:
 
 ## SageMaker Execution Role
 
-The bootstrap command creates an IAM role (`mlcc-sagemaker-execution-role`) with permissions for:
+The bootstrap command creates an IAM role (`mlcc-sagemaker-execution-role`) with these permission groups:
 
-- **SageMaker**: Create, update, delete, and invoke endpoints, endpoint configs, models, and inference components
-- **ECR**: Pull images from the `ml-container-creator` repository
-- **CloudWatch Logs**: Write container logs
-- **S3**: Read model artifacts from `ml-container-creator-*` buckets
+### Endpoint Management
+Create, update, delete, describe, and invoke endpoints, endpoint configs, models, and inference components.
 
-The role is defined in the CloudFormation stack template (`config/bootstrap-stack.json`) and updated automatically when you re-run bootstrap after upgrading.
+### AI Benchmarking
+Create, describe, list, stop, and delete AI benchmark jobs, AI recommendation jobs, and AI workload configs.
+
+### Training & Model Customization
+Create/describe/stop training jobs, model packages, model package groups. Access SageMaker Hub contents. Manage training plans.
+
+### MLflow Integration
+List/describe MLflow tracking servers and apps. Create presigned URLs. Call MLflow app APIs.
+
+### ECR
+Pull container images (GetAuthorizationToken, BatchGetImage, GetDownloadUrlForLayer, BatchCheckLayerAvailability).
+
+### S3
+Read and write model artifacts, adapters, benchmark results:
+- `s3:GetObject`, `s3:PutObject`, `s3:AbortMultipartUpload`, `s3:ListBucket`
+- Scoped to `mlcc-*` and `ml-container-creator-*` buckets
+
+### CloudWatch Logs
+Create log groups/streams and put log events.
+
+### Secrets Manager
+Read and write secrets prefixed with `mlcc/` or `ml-container-creator/` (used for HF tokens, API keys).
+
+### SNS
+Publish notifications to `mlcc-*` and `ml-container-creator-*` topics (benchmark completion alerts).
+
+### Service Quotas & Capacity
+Query service quotas and training plan availability for instance selection.
+
+### Lambda
+Invoke functions (reward model evaluation during training/tuning).
+
+### PassRole
+Self-pass to SageMaker service, scoped to `mlcc-sagemaker-execution-role`.
+
+The role is defined in `config/bootstrap-stack.json` and updated automatically when you re-run bootstrap after upgrading.
 
 If you use a custom role (`--role-arn`), ensure it has at minimum:
 
@@ -25,12 +58,15 @@ If you use a custom role (`--role-arn`), ensure it has at minimum:
 |-----------|---------|
 | `sagemaker:CreateEndpoint`, `CreateEndpointConfig`, `CreateModel`, `CreateInferenceComponent` | Deploy |
 | `sagemaker:DeleteEndpoint`, `DeleteEndpointConfig`, `DeleteModel`, `DeleteInferenceComponent` | Clean up |
-| `sagemaker:DescribeEndpoint`, `DescribeEndpointConfig`, `DescribeModel`, `DescribeInferenceComponent` | Status checks |
+| `sagemaker:DescribeEndpoint`, `DescribeEndpointConfig`, `DescribeModel`, `DescribeInferenceComponent`, `ListInferenceComponents` | Status |
 | `sagemaker:InvokeEndpoint`, `InvokeEndpointAsync` | Inference |
 | `sagemaker:UpdateEndpoint`, `UpdateEndpointWeightsAndCapacities`, `UpdateInferenceComponent` | Updates |
-| `ecr:GetAuthorizationToken`, `BatchGetImage`, `GetDownloadUrlForLayer`, `BatchCheckLayerAvailability` | Pull container image |
-| `logs:CreateLogGroup`, `CreateLogStream`, `PutLogEvents` | Container logging |
-| `s3:GetObject`, `s3:ListBucket` on `ml-container-creator-*` | Model artifact access |
+| `sagemaker:CreateAIBenchmarkJob`, `DescribeAIBenchmarkJob`, `ListAIBenchmarkJobs` | Benchmark |
+| `sagemaker:CreateTrainingJob`, `DescribeTrainingJob`, `StopTrainingJob` | Training/tuning |
+| `ecr:GetAuthorizationToken`, `BatchGetImage`, `GetDownloadUrlForLayer`, `BatchCheckLayerAvailability` | Pull image |
+| `logs:CreateLogGroup`, `CreateLogStream`, `PutLogEvents` | Logging |
+| `s3:GetObject`, `s3:PutObject`, `s3:ListBucket` on project buckets | Artifacts |
+| `iam:PassRole` (to sagemaker.amazonaws.com) | Role delegation |
 
 Trust policy must allow `sagemaker.amazonaws.com` to assume the role.
 
@@ -48,12 +84,27 @@ Your AWS user or CI system needs these permissions to run the do-scripts:
 
 | Script | Permissions Needed |
 |--------|-------------------|
+| `./do/build` | Local only — no AWS permissions |
+| `./do/run` | Local only — no AWS permissions |
 | `./do/push` | `ecr:GetAuthorizationToken`, `ecr:PutImage`, `ecr:InitiateLayerUpload`, `ecr:UploadLayerPart`, `ecr:CompleteLayerUpload`, `ecr:BatchCheckLayerAvailability` |
 | `./do/submit` | `codebuild:CreateProject`, `codebuild:StartBuild`, `codebuild:BatchGetBuilds`, `iam:CreateRole`, `iam:PutRolePolicy`, `iam:PassRole`, `s3:PutObject`, `s3:CreateBucket` |
-| `./do/deploy` | `sagemaker:CreateEndpointConfig`, `sagemaker:CreateEndpoint`, `sagemaker:CreateInferenceComponent`, `sagemaker:DescribeEndpoint`, `iam:PassRole` |
-| `./do/clean` | `sagemaker:DeleteEndpoint`, `sagemaker:DeleteEndpointConfig`, `sagemaker:DeleteInferenceComponent`, `codebuild:DeleteProject`, `iam:DeleteRole`, `iam:DeleteRolePolicy` |
+| `./do/stage` | `s3:PutObject`, `s3:GetObject`, `s3:ListBucket` on mlcc-* buckets |
+| `./do/deploy` | `sagemaker:CreateEndpointConfig`, `sagemaker:CreateEndpoint`, `sagemaker:CreateModel`, `sagemaker:CreateInferenceComponent`, `sagemaker:DescribeEndpoint`, `iam:PassRole` |
+| `./do/add-ic` | `sagemaker:CreateInferenceComponent`, `sagemaker:DescribeEndpoint`, `sagemaker:ListInferenceComponents`, `iam:PassRole` |
 | `./do/test` | `sagemaker-runtime:InvokeEndpoint` |
-| `bootstrap` | `cloudformation:*`, `iam:CreateRole`, `iam:PutRolePolicy`, `iam:TagRole`, `ecr:CreateRepository`, `s3:CreateBucket` (and `sts:GetCallerIdentity`) |
+| `./do/benchmark` | `sagemaker:CreateAIBenchmarkJob`, `sagemaker:DescribeAIBenchmarkJob`, `sagemaker:ListAIBenchmarkJobs`, `sagemaker:CreateAIWorkloadConfig`, `iam:PassRole`, `s3:GetObject` |
+| `./do/train` | `sagemaker:CreateTrainingJob`, `sagemaker:DescribeTrainingJob`, `iam:PassRole`, `s3:GetObject`, `s3:PutObject` |
+| `./do/tune` | `sagemaker:CreateTrainingJob`, `sagemaker:DescribeTrainingJob`, `iam:PassRole`, `s3:GetObject`, `s3:PutObject` |
+| `./do/adapter` | `sagemaker:CreateInferenceComponent`, `sagemaker:UpdateInferenceComponent`, `sagemaker:DescribeInferenceComponent`, `s3:GetObject` |
+| `./do/optimize` | `sagemaker:CreateModel`, `sagemaker:DescribeModel`, `s3:GetObject`, `s3:PutObject` |
+| `./do/register` | `sagemaker:CreateModelPackage`, `sagemaker:CreateModelPackageGroup`, `sagemaker:DescribeModelPackage` |
+| `./do/logs` | `logs:GetLogEvents`, `logs:FilterLogEvents`, `logs:DescribeLogStreams` |
+| `./do/status` | `sagemaker:DescribeEndpoint`, `sagemaker:DescribeInferenceComponent`, `sagemaker:ListInferenceComponents` |
+| `./do/clean` | `sagemaker:DeleteEndpoint`, `sagemaker:DeleteEndpointConfig`, `sagemaker:DeleteModel`, `sagemaker:DeleteInferenceComponent`, `codebuild:DeleteProject`, `iam:DeleteRole`, `iam:DeleteRolePolicy` |
+| `./do/export` | Local only — reads config files |
+| `./do/validate` | Local only — validates project structure |
+| `./do/manifest` | Local only — generates deployment manifest |
+| `bootstrap` | `cloudformation:*`, `iam:CreateRole`, `iam:PutRolePolicy`, `iam:TagRole`, `ecr:CreateRepository`, `s3:CreateBucket`, `sts:GetCallerIdentity` |
 
 <% if (framework === 'transformers' && hfToken) { %>
 ## HuggingFace Token Security