@aws/ml-container-creator 0.10.0 → 0.12.1

package/src/lib/bootstrap-command-handler.js

@@ -62,6 +62,7 @@ export default class BootstrapCommandHandler {
     _handleScan() { return this.profileManager._handleScan(); }
     _handlePrune() { return this.profileManager._handlePrune(); }
     _handleSyncSchemas() { return this.profileManager._handleSyncSchemas(); }
+    _handleSyncModelFamilies() { return this.profileManager._handleSyncModelFamilies(); }
 
     /**
      * Dispatch bootstrap subcommands.
@@ -69,8 +70,28 @@ export default class BootstrapCommandHandler {
      * @param {object} options - Parsed CLI options
      */
     async handle(args, options) {
+        // Commander.js with passThroughOptions() captures flags after positional
+        // arguments in args rather than options. Extract known flags from args.
+        const extractedOptions = { ...options };
+        const cleanArgs = [];
+        for (const arg of args) {
+            if (arg === '--ci') extractedOptions.ci = true;
+            else if (arg === '--benchmark-infra') extractedOptions.benchmarkInfra = true;
+            else if (arg === '--skip-ci') extractedOptions.skipCi = true;
+            else if (arg === '--skip-s3') extractedOptions.skipS3 = true;
+            else if (arg === '--skip-post-setup') extractedOptions.skipPostSetup = true;
+            else if (arg === '--force') extractedOptions.force = true;
+            else if (arg === '--verify') extractedOptions.verify = true;
+            else if (arg === '--delete-stack') extractedOptions.deleteStack = true;
+            else if (arg === '--non-interactive') extractedOptions.nonInteractive = true;
+            else if (arg === '--ignore-staleness') extractedOptions.ignoreStaleness = true;
+            else cleanArgs.push(arg);
+        }
+        args = cleanArgs;
+        options = extractedOptions;
+
         // Handle legacy --sync-schemas flag for backward compatibility
-        if (options['sync-schemas']) {
+        if ((options['sync-schemas'] || options.syncSchemas)) {
             await this._handleSyncSchemas();
             if (args.length === 0) return;
         }
@@ -107,6 +128,15 @@ export default class BootstrapCommandHandler {
         case 'sync-schemas':
             await this._handleSyncSchemas();
             break;
+        case 'sync-model-families':
+            await this._handleSyncModelFamilies();
+            break;
+        // Migration path: upgrades legacy profiles to current naming conventions.
+        // Corrects stackName to mlcc-bootstrap-{profileName}, renames sharedStackFrom
+        // to sharedInfraFrom. Idempotent — safe to run multiple times.
+        case 'migrate':
+            await this._handleMigrate();
+            break;
         default:
             console.log(`Unknown bootstrap subcommand: ${subcommand}`);
             this._showHelp();
@@ -119,7 +149,8 @@ export default class BootstrapCommandHandler {
      * @param {object} options - Parsed CLI options
      */
     async _handleInteractiveSetup(options) {
-        const nonInteractive = options['non-interactive'];
+        // Commander.js converts --non-interactive to options.nonInteractive (camelCase)
+        const nonInteractive = options['non-interactive'] || options.nonInteractive;
 
         // Non-interactive mode: validate required flags upfront
         if (nonInteractive) {
@@ -180,13 +211,13 @@ export default class BootstrapCommandHandler {
 
         // Step 3: Determine stack parameters
         let useExistingRoleArn = '';
-        if (nonInteractive && options['role-arn']) {
-            useExistingRoleArn = options['role-arn'];
-            console.log(`  Using provided IAM role ARN: ${options['role-arn']}`);
+        if (nonInteractive && (options['role-arn'] || options.roleArn)) {
+            useExistingRoleArn = (options['role-arn'] || options.roleArn);
+            console.log(`  Using provided IAM role ARN: ${(options['role-arn'] || options.roleArn)}`);
         }
 
         let createS3Buckets = false;
-        if (nonInteractive && options['skip-s3']) {
+        if (nonInteractive && (options['skip-s3'] || options.skipS3)) {
             console.log('  ⏭️  Skipping S3 bucket creation (--skip-s3)');
         } else if (nonInteractive) {
             createS3Buckets = true;
@@ -231,7 +262,8 @@ export default class BootstrapCommandHandler {
 
                 profileData.roleArn = stackOutputs.RoleArn;
                 profileData.ecrRepositoryName = stackOutputs.EcrRepositoryName;
-                profileData.stackName = otherStack;
+                profileData.stackName = stackName;
+                profileData.sharedInfraFrom = otherStack;  // Track that this profile reuses another's stack
                 if (stackOutputs.AsyncS3BucketName) profileData.asyncS3Bucket = stackOutputs.AsyncS3BucketName;
                 if (stackOutputs.BatchS3BucketName) profileData.batchS3Bucket = stackOutputs.BatchS3BucketName;
                 if (stackOutputs.AdapterS3BucketName) profileData.adapterS3Bucket = stackOutputs.AdapterS3BucketName;
@@ -245,15 +277,45 @@ export default class BootstrapCommandHandler {
         }
 
         if (!profileData.stackName) {
+            // Pre-check: if IAM role already exists globally (from another region's deployment),
+            // pass its ARN so CloudFormation skips re-creation (account-level singleton)
+            if (!useExistingRoleArn) {
+                try {
+                    const roleResult = this._execAws(
+                        'iam get-role --role-name mlcc-sagemaker-execution-role',
+                        awsProfile
+                    );
+                    const roleArn = roleResult && roleResult.Role && roleResult.Role.Arn;
+                    if (roleArn && roleArn.startsWith('arn:aws:iam::')) {
+                        useExistingRoleArn = roleArn;
+                        console.log(`  ℹ️  Reusing existing IAM role: ${roleArn}`);
+                    }
+                } catch (_) {
+                    // Role doesn't exist yet — will be created by the stack
+                }
+            }
+
             try {
+                // Check if ECR repo already exists (avoid ResourceExistenceCheck failure)
+                let skipEcr = 'false';
+                try {
+                    this._execAws(
+                        `ecr describe-repositories --repository-names ml-container-creator --region ${region}`,
+                        awsProfile
+                    );
+                    skipEcr = 'true';
+                    console.log('  ℹ️  ECR repository already exists — skipping creation');
+                } catch (_) { /* doesn't exist — will be created */ }
+
                 const stackOutputs = this._deployStack(stackName, {
                     CreateS3Buckets: createS3Buckets ? 'true' : 'false',
-                    UseExistingRoleArn: useExistingRoleArn
+                    UseExistingRoleArn: useExistingRoleArn,
+                    SkipEcrCreation: skipEcr
                 }, awsProfile, region);
 
                 // Read outputs into profile data
                 profileData.roleArn = stackOutputs.RoleArn;
-                profileData.ecrRepositoryName = stackOutputs.EcrRepositoryName;
+                profileData.ecrRepositoryName = stackOutputs.EcrRepositoryName || 'ml-container-creator';
                 profileData.stackName = stackName;
 
                 if (stackOutputs.AsyncS3BucketName) {
@@ -278,6 +340,23 @@ export default class BootstrapCommandHandler {
             }
         } // end if (!profileData.stackName)
 
+        // Step 4b: MLflow App for model customization experiment tracking
+        this._displayProgress('📊', 'MLflow App for experiment tracking...');
+        try {
+            if (!profileData.mlflowAppArn) {
+                const mlflowAppArn = this._ensureMlflowApp(profileData, awsProfile);
+                if (mlflowAppArn) {
+                    profileData.mlflowAppArn = mlflowAppArn;
+                    console.log(`  ✅ MLflow App ready: ${mlflowAppArn}`);
+                }
+            } else {
+                console.log(`  ✅ MLflow App already configured: ${profileData.mlflowAppArn}`);
+            }
+        } catch (error) {
+            console.log(`  ⚠️  MLflow App setup skipped: ${error.message}`);
+            console.log('     Tune jobs will still work but experiment tracking may not be available.');
+        }
+
         // Step 5: CI Infrastructure setup (separate CDK stack — unchanged)
         this._displayProgress('🧪', 'CI Testing Infrastructure...');
         try {
@@ -286,7 +365,7 @@ export default class BootstrapCommandHandler {
             if (nonInteractive) {
                 if (options.ci) {
                     provisionCi = true;
-                } else if (options['skip-ci']) {
+                } else if ((options['skip-ci'] || options.skipCi)) {
                     console.log('  ⏭️  Skipping CI infrastructure (--skip-ci)');
                     provisionCi = false;
                 } else {
@@ -303,6 +382,21 @@ export default class BootstrapCommandHandler {
             }
 
             if (provisionCi) {
+                // --- CI single-region enforcement ---
+                const ciConflict = this._findExistingCiProfile(profileName);
+                if (ciConflict) {
+                    console.log(`❌ CI infrastructure already deployed in region ${ciConflict.config.awsRegion} (profile: ${ciConflict.name}).`);
+                    console.log('   CI can only be deployed in one region per account.');
+                    provisionCi = false;
+                }
+            }
+
+            if (provisionCi) {
+                // Persist CI intent immediately so that `bootstrap update --ci` can
+                // retry if the CDK deploy fails. Don't wait for success.
+                profileData.ciInfraProvisioned = true;
+                profileData.ciTableName = profileData.ciTableName || 'mlcc-ci-table';
+
                 // Ensure CDK is bootstrapped in this account/region
                 const cdkBootstrapped = this._resourceExists(
                     `ssm get-parameter --name /cdk-bootstrap/hnb659fds/version --region ${profileData.awsRegion}`,
@@ -358,14 +452,39 @@ export default class BootstrapCommandHandler {
                         stdio: ['pipe', 'pipe', 'pipe']
                     });
 
+                    // Warn if shell AWS_REGION differs from profile region
+                    if (process.env.AWS_REGION && process.env.AWS_REGION !== profileData.awsRegion) {
+                        console.log(`  ⚠️  AWS_REGION env var (${process.env.AWS_REGION}) differs from profile region (${profileData.awsRegion}) — using profile region`);
+                    }
+
+                    // --no-rollback prevents rollback on AlreadyExists errors for IAM roles
+                    // that may pre-exist from a prior deployment or another region.
+                    // Check if benchmark bucket already exists (from a prior torn-down stack with RETAIN policy)
+                    let importBucketCtx = '';
+                    if (options.benchmarkInfra) {
+                        try {
+                            execSync(
+                                `aws s3api head-bucket --bucket mlcc-benchmark-results-${profileData.accountId}-${profileData.awsRegion}${profileData.awsProfile ? ` --profile ${profileData.awsProfile}` : ''} --region ${profileData.awsRegion}`,
+                                { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }
+                            );
+                            importBucketCtx = ' -c importExistingBenchmarkBucket=true';
+                            console.log('  ℹ️  Benchmark results bucket already exists — importing into stack');
+                        } catch {
+                            // Bucket doesn't exist — will be created fresh
+                        }
+                    }
+                    const cdkDeployCmd = options.benchmarkInfra
+                        ? `npx cdk deploy MlccCiHarnessStack --require-approval never --no-rollback --parameters MlccCiHarnessStack:CreateBenchmarkInfra=true${importBucketCtx}`
+                        : 'npx cdk deploy MlccCiHarnessStack --require-approval never --no-rollback';
                     execSync(
-                        'npx cdk deploy MlccCiHarnessStack --require-approval never',
+                        cdkDeployCmd,
                         {
                             cwd: ciHarnessDir,
                             encoding: 'utf8',
                             stdio: 'inherit',
                             env: {
                                 ...process.env,
+                                AWS_REGION: profileData.awsRegion,
                                 CDK_DEFAULT_REGION: profileData.awsRegion,
                                 CDK_DEFAULT_ACCOUNT: profileData.accountId,
                                 AWS_PROFILE: profileData.awsProfile
@@ -376,6 +495,11 @@ export default class BootstrapCommandHandler {
 
                     profileData.ciInfraProvisioned = true;
                     profileData.ciTableName = 'mlcc-ci-table';
+                    if (options.benchmarkInfra) {
+                        profileData.benchmarkInfraProvisioned = true;
+                        profileData.ciGlueDatabase = 'mlcc_ci';
+                        profileData.ciBenchmarkResultsBucket = `mlcc-benchmark-results-${profileData.accountId}-${profileData.awsRegion}`;
+                    }
                 }
             }
         } catch (error) {
@@ -413,14 +537,80 @@ export default class BootstrapCommandHandler {
         console.log(`   Region: ${profileConfig.awsRegion}`);
         console.log(`   Account: ${profileConfig.accountId}`);
 
+        // --- SANITY CHECK 1: Account identity ---
+        const callerAccount = this._getCallerAccount(profileConfig.awsProfile);
+        if (callerAccount !== profileConfig.accountId) {
+            console.log(`❌ Account mismatch: profile expects ${profileConfig.accountId} but credentials resolve to ${callerAccount}`);
+            return;
+        }
+
         // Re-deploy the CloudFormation bootstrap stack
         const stackName = profileConfig.stackName || `${STACK_NAME_PREFIX}-${name}`;
+
+        // Sanity check: stack name consistency (warn-and-continue)
+        const expectedStackName = `${STACK_NAME_PREFIX}-${name}`;
+        if (profileConfig.stackName && profileConfig.stackName !== expectedStackName) {
+            console.log(`⚠️  Stack name mismatch: expected "${expectedStackName}" but profile has "${profileConfig.stackName}"`);
+            console.log('   Run `ml-container-creator bootstrap migrate` to fix.');
+            console.log('   Proceeding with stored stack name...');
+        }
+
+        // --- SANITY CHECK 3: Stack exists in target region ---
+        const stackExists = this._resourceExists(
+            `cloudformation describe-stacks --stack-name ${stackName} --region ${profileConfig.awsRegion}`,
+            profileConfig.awsProfile
+        );
+        if (!stackExists) {
+            console.log(`❌ Stack "${stackName}" not found in ${profileConfig.awsRegion}.`);
+            console.log('   Run `ml-container-creator bootstrap` to create it.');
+            return;
+        }
+
+        // --- CI single-region enforcement ---
+        if (options.ci) {
+            const ciConflict = this._findExistingCiProfile(name);
+            if (ciConflict) {
+                console.log(`❌ CI infrastructure already deployed in region ${ciConflict.config.awsRegion} (profile: ${ciConflict.name}).`);
+                console.log('   CI can only be deployed in one region per account.');
+                return;
+            }
+        }
+
         this._displayProgress('☁️', 'Updating bootstrap stack...');
 
+        // Pre-check: if IAM role already exists globally (from another region's deployment),
+        // pass its ARN so CloudFormation skips re-creation (account-level singleton)
+        let useExistingRoleArn = profileConfig.roleArn || '';
+        if (!useExistingRoleArn) {
+            try {
+                const roleResult = this._execAws(
+                    'iam get-role --role-name mlcc-sagemaker-execution-role',
+                    profileConfig.awsProfile
+                );
+                const roleArn = roleResult && roleResult.Role && roleResult.Role.Arn;
+                if (roleArn && roleArn.startsWith('arn:aws:iam::')) {
+                    useExistingRoleArn = roleArn;
+                }
+            } catch (_) {
+                // Role doesn't exist yet — will be created by the stack
+            }
+        }
+
         try {
+            // Check if ECR repo already exists (avoid ResourceExistenceCheck failure)
+            let skipEcr = 'false';
+            try {
+                this._execAws(
+                    `ecr describe-repositories --repository-names ml-container-creator --region ${profileConfig.awsRegion}`,
+                    profileConfig.awsProfile
+                );
+                skipEcr = 'true';
+            } catch (_) { /* doesn't exist */ }
+
             const stackOutputs = this._deployStack(stackName, {
                 CreateS3Buckets: (profileConfig.asyncS3Bucket || profileConfig.batchS3Bucket) ? 'true' : 'false',
-                UseExistingRoleArn: ''
+                UseExistingRoleArn: useExistingRoleArn,
+                SkipEcrCreation: skipEcr
             }, profileConfig.awsProfile, profileConfig.awsRegion);
 
             // Update profile with any new outputs
@@ -456,14 +646,34 @@ export default class BootstrapCommandHandler {
                         stdio: ['pipe', 'pipe', 'pipe']
                     });
 
+                    // --no-rollback prevents rollback on AlreadyExists errors for IAM roles
+                    // that may pre-exist from a prior deployment or another region.
+                    // Check if benchmark bucket already exists (from a prior torn-down stack with RETAIN policy)
+                    let updateImportBucketCtx = '';
+                    if (options.benchmarkInfra || profileConfig.benchmarkInfraProvisioned) {
+                        try {
+                            execSync(
+                                `aws s3api head-bucket --bucket mlcc-benchmark-results-${profileConfig.accountId}-${profileConfig.awsRegion}${profileConfig.awsProfile ? ` --profile ${profileConfig.awsProfile}` : ''} --region ${profileConfig.awsRegion}`,
+                                { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }
+                            );
+                            updateImportBucketCtx = ' -c importExistingBenchmarkBucket=true';
+                            console.log('  ℹ️  Benchmark results bucket already exists — importing into stack');
+                        } catch {
+                            // Bucket doesn't exist — will be created fresh
+                        }
+                    }
+                    const updateCdkCmd = (options.benchmarkInfra || profileConfig.benchmarkInfraProvisioned)
+                        ? `npx cdk deploy MlccCiHarnessStack --require-approval never --no-rollback --parameters MlccCiHarnessStack:CreateBenchmarkInfra=true${updateImportBucketCtx}`
+                        : 'npx cdk deploy MlccCiHarnessStack --require-approval never --no-rollback';
                     execSync(
-                        'npx cdk deploy MlccCiHarnessStack --require-approval never',
+                        updateCdkCmd,
                         {
                             cwd: ciHarnessDir,
                             encoding: 'utf8',
                             stdio: 'inherit',
                             env: {
                                 ...process.env,
+                                AWS_REGION: profileConfig.awsRegion,
                                 CDK_DEFAULT_REGION: profileConfig.awsRegion,
                                 CDK_DEFAULT_ACCOUNT: profileConfig.accountId,
                                 AWS_PROFILE: profileConfig.awsProfile
@@ -471,6 +681,8 @@ export default class BootstrapCommandHandler {
                         }
                     );
                     profileConfig.ciInfraProvisioned = true;
+                    profileConfig.ciGlueDatabase = profileConfig.ciGlueDatabase || 'mlcc_ci';
+                    profileConfig.ciBenchmarkResultsBucket = profileConfig.ciBenchmarkResultsBucket || `mlcc-benchmark-results-${profileConfig.accountId}-${profileConfig.awsRegion}`;
                     console.log('  ✅ CI harness stack updated');
                 }
             } catch (error) {
@@ -480,6 +692,18 @@ export default class BootstrapCommandHandler {
             console.log('  ⏭️  CI stack skipped (not provisioned — use --ci to force)');
         }
 
+        // Ensure MLflow App exists
+        this._displayProgress('📊', 'MLflow App for experiment tracking...');
+        try {
+            const mlflowAppArn = this._ensureMlflowApp(profileConfig, profileConfig.awsProfile);
+            if (mlflowAppArn) {
+                profileConfig.mlflowAppArn = mlflowAppArn;
+                console.log(`  ✅ MLflow App ready: ${mlflowAppArn}`);
+            }
+        } catch (error) {
+            console.log(`  ⚠️  MLflow App setup skipped: ${error.message}`);
+        }
+
         // Save updated profile
         this.config.setProfile(name, profileConfig);
         console.log(`\n✅ Update complete for profile "${name}"`);
@@ -488,6 +712,82 @@ export default class BootstrapCommandHandler {
         await this._runPostSetupChain(options);
     }
 
+    /**
+     * Migrate legacy profiles to current naming conventions.
+     * Corrects stackName mismatches and renames sharedStackFrom → sharedInfraFrom.
+     * Displays a preview of all changes and requires confirmation before writing.
+     */
+    async _handleMigrate() {
+        const config = this.config.read();
+        if (!config || !config.profiles) {
+            console.log('No profiles to migrate.');
+            return;
+        }
+
+        const changes = [];
+
+        for (const [name, profileConfig] of Object.entries(config.profiles)) {
+            const expected = `${STACK_NAME_PREFIX}-${name}`;
+
+            // Fix stackName mismatch
+            if (profileConfig.stackName && profileConfig.stackName !== expected) {
+                changes.push({
+                    profile: name,
+                    field: 'stackName',
+                    from: profileConfig.stackName,
+                    to: expected
+                });
+            }
+
+            // Rename sharedStackFrom → sharedInfraFrom
+            if (profileConfig.sharedStackFrom) {
+                changes.push({
+                    profile: name,
+                    field: 'sharedStackFrom → sharedInfraFrom',
+                    from: profileConfig.sharedStackFrom,
+                    to: profileConfig.sharedStackFrom
+                });
+            }
+        }
+
+        if (changes.length === 0) {
+            console.log('✅ All profiles already use current naming conventions.');
+            return;
+        }
+
+        // Display preview
+        console.log('📋 Migration Preview:\n');
+        for (const change of changes) {
+            console.log(`  Profile "${change.profile}":`);
+            console.log(`    ${change.field}: "${change.from}" → "${change.to}"`);
+        }
+
+        // Prompt for confirmation
+        const { confirm } = await this._promptFn([{
+            type: 'confirm',
+            name: 'confirm',
+            message: 'Apply these changes?',
+            default: true
+        }]);
+
+        if (!confirm) return;
+
+        // Apply changes
+        for (const [name, profileConfig] of Object.entries(config.profiles)) {
+            const expected = `${STACK_NAME_PREFIX}-${name}`;
+            if (profileConfig.stackName !== expected) {
+                profileConfig.stackName = expected;
+            }
+            if (profileConfig.sharedStackFrom) {
+                profileConfig.sharedInfraFrom = profileConfig.sharedStackFrom;
+                delete profileConfig.sharedStackFrom;
+            }
+        }
+
+        this.config.write(config);
+        console.log('✅ Migration complete.');
+    }
+
     /**
      * Run the post-setup chain: mcp init → registry sync-architectures → sync-schemas.
      * Each step is independent — failures are collected and reported at the end.
@@ -495,7 +795,7 @@ export default class BootstrapCommandHandler {
      * @param {object} options - Parsed CLI options (checks skipPostSetup)
      */
     async _runPostSetupChain(options = {}) {
-        if (options['skip-post-setup']) {
+        if ((options['skip-post-setup'] || options.skipPostSetup)) {
             console.log('\n⏭️  Skipping post-setup chain (--skip-post-setup)');
             return;
         }
@@ -717,7 +1017,8 @@ export default class BootstrapCommandHandler {
      * @returns {object} Parsed JSON output
      */
     _execAws(command, profile) {
-        const fullCommand = `aws ${command} --profile ${profile} --output json`;
+        const profileFlag = profile ? `--profile ${profile}` : '';
+        const fullCommand = `aws ${command} ${profileFlag} --output json`.replace(/\s+/g, ' ').trim();
         const output = execSync(fullCommand, { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] });
         const trimmed = output.trim();
         if (!trimmed) {
@@ -729,6 +1030,12 @@ export default class BootstrapCommandHandler {
     /**
      * Deploy the bootstrap CloudFormation stack and return its outputs.
      *
+     * Before deploying, checks for pre-existing S3 buckets that would cause
+     * ResourceExistenceCheck failures. If the stack is in REVIEW_IN_PROGRESS
+     * state (empty shell from a failed prior attempt), deletes it first.
+     * If buckets exist but aren't managed by the stack, uses a CloudFormation
+     * import changeset to adopt them before proceeding with the normal deploy.
+     *
      * Uses `aws cloudformation deploy` which is idempotent — it creates the
      * stack on first run and updates it on subsequent runs. If the template
      * hasn't changed, it exits with "No changes to deploy" which we handle
@@ -741,6 +1048,9 @@ export default class BootstrapCommandHandler {
      * @returns {object} Map of output key → output value
      */
     _deployStack(stackName, parameters, profile, region) {
+        // Handle ghost stacks and pre-existing resources
+        this._resolveStackConflicts(stackName, parameters, profile, region);
+
         // Build parameter overrides string
         const paramOverrides = Object.entries(parameters)
             .map(([key, value]) => `${key}=${value}`)
@@ -764,6 +1074,32 @@ export default class BootstrapCommandHandler {
             const stderr = error.stderr || error.message || '';
             if (stderr.includes('No changes to deploy')) {
                 console.log('  ℹ️  Stack is up to date — no changes needed');
+            } else if (stderr.includes('ResourceExistenceCheck')) {
+                // Resources already exist outside the stack — attempt import and retry
+                console.log('  ⚠️  Pre-existing resources detected — attempting import...');
+                this._resolveStackConflicts(stackName, parameters, profile, region);
+                // Rebuild deploy command with updated parameters (e.g., CreateS3Buckets may now be 'false')
+                const retryParamOverrides = Object.entries(parameters)
+                    .map(([key, value]) => `${key}=${value}`)
+                    .join(' ');
+                const retryDeployCommand = [
+                    'aws cloudformation deploy',
+                    `--template-file ${STACK_TEMPLATE_PATH}`,
+                    `--stack-name ${stackName}`,
+                    '--capabilities CAPABILITY_NAMED_IAM',
+                    `--parameter-overrides ${retryParamOverrides}`,
+                    `--profile ${profile}`,
+                    `--region ${region}`
+                ].join(' ');
+                // Retry the deploy after import
+                try {
+                    execSync(retryDeployCommand, { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] });
+                } catch (retryError) {
+                    const retryStderr = retryError.stderr || retryError.message || '';
+                    if (!retryStderr.includes('No changes to deploy')) {
+                        throw retryError;
+                    }
+                }
             } else {
                 throw error;
             }
@@ -785,9 +1121,144 @@ export default class BootstrapCommandHandler {
             outputs[output.OutputKey] = output.OutputValue;
         }
 
+        // If S3 buckets already existed (skipped creation), inject their names
+        // into outputs so the profile config gets populated correctly.
+        if (this._preExistingBuckets && this._preExistingBuckets.length > 0) {
+            const bucketOutputMap = {
+                'AsyncS3Bucket': 'AsyncS3BucketName',
+                'BatchS3Bucket': 'BatchS3BucketName',
+                'AdapterS3Bucket': 'AdapterS3BucketName',
+                'BenchmarkS3Bucket': 'BenchmarkS3BucketName',
+                'TuneS3Bucket': 'TuneS3BucketName'
+            };
+            for (const bucket of this._preExistingBuckets) {
+                const outputKey = bucketOutputMap[bucket.logicalId];
+                if (outputKey && !outputs[outputKey]) {
+                    outputs[outputKey] = bucket.name;
+                }
+            }
+            this._preExistingBuckets = null;
+        }
+
         return outputs;
     }
 
+    /**
+     * Resolve stack conflicts before deploying.
+     *
+     * Handles two scenarios that cause ResourceExistenceCheck failures:
+     * 1. Ghost stacks (REVIEW_IN_PROGRESS) — delete them first
+     * 2. Pre-existing S3 buckets not managed by the stack — import them
+     *
+     * @param {string} stackName - CloudFormation stack name
+     * @param {object} parameters - Stack parameter key-value pairs
+     * @param {string} profile - AWS CLI profile name
+     * @param {string} region - AWS region
+     */
+    _resolveStackConflicts(stackName, parameters, profile, region) {
+        // Check if stack exists and its status
+        let stackStatus = null;
+        let managedResources = [];
+
+        try {
+            const describeResult = this._execAws(
+                `cloudformation describe-stacks --stack-name ${stackName} --region ${region}`,
+                profile
+            );
+            const stack = describeResult.Stacks && describeResult.Stacks[0];
+            if (stack) {
+                stackStatus = stack.StackStatus;
+            }
+        } catch (_) {
+            // Stack doesn't exist — no conflicts possible
+            return;
+        }
+
+        // Handle ghost stacks (created but never successfully deployed)
+        if (stackStatus === 'REVIEW_IN_PROGRESS') {
+            console.log('  ⚠️  Found ghost stack (REVIEW_IN_PROGRESS) — deleting before redeploy...');
+            try {
+                execSync(
+                    `aws cloudformation delete-stack --stack-name ${stackName} --profile ${profile} --region ${region}`,
+                    { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }
+                );
+                execSync(
+                    `aws cloudformation wait stack-delete-complete --stack-name ${stackName} --profile ${profile} --region ${region}`,
+                    { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 60000 }
+                );
+                console.log('  ✅ Ghost stack deleted');
+            } catch (err) {
+                console.log(`  ⚠️  Could not delete ghost stack: ${err.message || err}`);
+            }
+            // Don't return — fall through to check for pre-existing S3 buckets
+            // that need to be imported on the fresh deploy. The ghost stack had
+            // DeletionPolicy:Retain buckets that survive stack deletion.
+            stackStatus = null;
+            managedResources = [];
+        }
+
+        // For active stacks (or post-ghost-deletion), check if S3 buckets exist but aren't managed
+        if (parameters.CreateS3Buckets !== 'true') {
+            return; // Not creating buckets — no conflict
+        }
+
+        // Get list of resources currently managed by the stack (empty if stack was just deleted)
+        if (stackStatus) {
+            try {
+                const resources = this._execAws(
+                    `cloudformation list-stack-resources --stack-name ${stackName} --region ${region}`,
+                    profile
+                );
+                managedResources = (resources.StackResourceSummaries || [])
+                    .map(r => r.LogicalResourceId);
+            } catch (_) {
+                // Stack doesn't exist or can't be queried — proceed with empty managedResources
+            }
+        }
+
+        // Check each S3 bucket that the template would create
+        const accountId = this._currentAccountId;
+        const bucketConfigs = [
+            { logicalId: 'AsyncS3Bucket', name: `mlcc-async-${accountId}-${region}` },
+            { logicalId: 'BatchS3Bucket', name: `mlcc-batch-${accountId}-${region}` },
+            { logicalId: 'AdapterS3Bucket', name: `mlcc-adapters-${accountId}-${region}` },
+            { logicalId: 'BenchmarkS3Bucket', name: `mlcc-benchmark-${accountId}-${region}` },
+            { logicalId: 'TuneS3Bucket', name: `mlcc-tune-${accountId}-${region}` }
+        ];
+
+        const bucketsToImport = [];
+
+        for (const bucket of bucketConfigs) {
+            if (managedResources.includes(bucket.logicalId)) {
+                continue; // Already managed by the stack — no conflict
+            }
+            // Check if bucket exists in AWS
+            try {
+                execSync(
+                    `aws s3api head-bucket --bucket ${bucket.name} --profile ${profile} --region ${region}`,
+                    { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }
+                );
+                // Bucket exists but not in stack — needs import
+                bucketsToImport.push(bucket);
+            } catch (_) {
+                // Bucket doesn't exist — will be created normally
+            }
+        }
+
+        if (bucketsToImport.length > 0) {
+            console.log(`  ℹ️  ${bucketsToImport.length} pre-existing S3 bucket(s) detected — skipping S3 creation (buckets already exist)`);
+
+            // Pre-existing S3 buckets survive stack deletion (DeletionPolicy: Retain).
+            // Rather than fighting CloudFormation's IMPORT limitations, just skip S3
+            // creation and wire the existing bucket names into the profile config directly.
+            // The naming convention is deterministic, so we know exactly what they are.
+            this._preExistingBuckets = bucketsToImport;
+
+            // Modify the parameters to skip S3 bucket creation in the deploy
+            parameters.CreateS3Buckets = 'false';
+        }
+    }
+
     /**
      * Write a JSON object to a temp file and return the `file://` path.
      * Used for passing complex JSON to AWS CLI commands without shell escaping issues.
@@ -821,6 +1292,125 @@ export default class BootstrapCommandHandler {
         }
     }
 
+    /**
+     * Get the AWS account ID from the caller's credentials.
+     * Uses `sts get-caller-identity` to resolve the actual account.
+     *
+     * @param {string} awsProfile - AWS CLI profile name
+     * @returns {string} The 12-digit AWS account ID
+     */
+    _getCallerAccount(awsProfile) {
+        const identity = this._execAws('sts get-caller-identity', awsProfile);
+        return identity.Account;
+    }
+
+    /**
+     * Scan all profiles to find one with ciInfraProvisioned=true,
+     * excluding the given profile name.
+     *
+     * @param {string} excludeProfile - Profile name to exclude from the scan
+     * @returns {{ name: string, config: Object }|null} The CI profile, or null if none found
+     */
+    _findExistingCiProfile(excludeProfile) {
+        const config = this.config.read();
+        if (!config || !config.profiles) return null;
+
+        for (const [name, profileConfig] of Object.entries(config.profiles)) {
+            if (name === excludeProfile) continue;
+            if (profileConfig.ciInfraProvisioned) {
+                return { name, config: profileConfig };
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Ensure an MLCC-owned MLflow App exists for experiment tracking.
+     * Creates one if it doesn't exist, using the tune S3 bucket as artifact store.
+     *
+     * @param {object} profileData - Bootstrap profile data (needs roleArn, awsRegion, accountId)
+     * @param {string} awsProfile - AWS CLI profile name
+     * @returns {string|null} MLflow App ARN or null if creation failed
+     */
+    _ensureMlflowApp(profileData, awsProfile) {
+        const region = profileData.awsRegion;
+        const accountId = profileData.accountId;
+        const roleArn = profileData.roleArn;
+        const appName = 'mlcc-tune-tracking';
+        const artifactBucket = `mlcc-tune-${accountId}-${region}`;
+
+        // Check if MLCC app already exists
+        try {
+            const apps = this._execAws(
+                `sagemaker list-mlflow-apps --region ${region}`,
+                awsProfile
+            );
+            const summaries = apps.Summaries || [];
+            const existing = summaries.find(a => a.Name === appName);
+            if (existing) {
+                return existing.Arn;
+            }
+        } catch {
+            // list-mlflow-apps may not be available in all CLI versions — proceed to create
+        }
+
+        // Create the MLflow App
+        console.log(`  Creating MLflow App "${appName}" with artifact store s3://${artifactBucket}...`);
+
+        // Ensure the artifact bucket exists (it's the tune bucket from the stack)
+        try {
+            this._execAws(
+                `s3api head-bucket --bucket ${artifactBucket} --region ${region}`,
+                awsProfile
+            );
+        } catch {
+            // Bucket doesn't exist — create it
+            console.log(`  Creating artifact bucket: ${artifactBucket}`);
+            try {
+                this._execAws(
+                    `s3api create-bucket --bucket ${artifactBucket} --region ${region} --create-bucket-configuration LocationConstraint=${region}`,
+                    awsProfile
+                );
+            } catch (bucketErr) {
+                // May already exist or region doesn't need LocationConstraint (us-east-1)
+                if (!bucketErr.message?.includes('BucketAlreadyOwnedByYou')) {
+                    try {
+                        this._execAws(
+                            `s3api create-bucket --bucket ${artifactBucket} --region ${region}`,
+                            awsProfile
+                        );
+                    } catch {
+                        // Bucket likely exists, continue
+                    }
+                }
+            }
+        }
+
+        // Create the app
+        try {
+            const result = this._execAws(
+                `sagemaker create-mlflow-app --name ${appName} --artifact-store-uri s3://${artifactBucket} --role-arn ${roleArn} --model-registration-mode AutoModelRegistrationEnabled --region ${region}`,
+                awsProfile
+            );
+            return result.Arn;
+        } catch (err) {
+            // If app already exists (race condition), try to describe it
+            if (err.message?.includes('ResourceLimitExceeded') || err.message?.includes('already exists')) {
+                try {
+                    const apps = this._execAws(
+                        `sagemaker list-mlflow-apps --region ${region}`,
+                        awsProfile
+                    );
+                    const found = (apps.Summaries || []).find(a => a.Name === appName);
+                    if (found) return found.Arn;
+                } catch {
+                    // Fall through
+                }
+            }
+            throw err;
+        }
+    }
+
     /**
      * Format tags for the AWS CLI --tags parameter.
      * Writes tags to a temp file and returns the file:// reference
@@ -858,6 +1448,8 @@ SUBCOMMANDS:
   scan                                Discover pre-existing MLCC-managed resources in AWS
   prune                               Remove deleted and unknown records from the deployment manifest
   update                              Re-deploy bootstrap stacks using active profile (no prompts)
+  migrate                             Upgrade legacy profiles to current naming conventions
+  sync-model-families                 Discover tune-eligible models from JumpStart Hub and update catalog
 
 SETUP OPTIONS:
   --non-interactive                   Run without interactive prompts
@@ -886,6 +1478,8 @@ EXAMPLES:
   ml-container-creator bootstrap remove dev
   ml-container-creator bootstrap remove dev --force --delete-stack
   ml-container-creator bootstrap scan
+  ml-container-creator bootstrap sync-model-families
+  ml-container-creator bootstrap migrate
   ml-container-creator bootstrap --non-interactive --profile my-aws-profile --region us-west-2
   ml-container-creator bootstrap --non-interactive --profile my-aws-profile --role-arn arn:aws:iam::123456789012:role/MyRole --skip-s3
   ml-container-creator bootstrap --non-interactive --profile my-aws-profile --region us-west-2 --ci