@aws/ml-container-creator 0.4.0 → 0.6.0

package/src/app.js

@@ -302,6 +302,22 @@ export async function writeProject(templateDir, destDir, answers, registryConfig
         ignorePatterns.push('**/hyperpod/**');
     }
 
+    // HyperPod is kubectl-based — no shared bash helpers or IC configs
+    if (answers.deploymentTarget === 'hyperpod-eks') {
+        ignorePatterns.push('**/do/lib/**');
+        ignorePatterns.push('**/do/ic/**');
+        ignorePatterns.push('**/do/add-ic');
+        ignorePatterns.push('**/do/status');
+        ignorePatterns.push('**/do/optimize');
+    }
+
+    // Async and batch don't use inference components (IC is real-time only)
+    if (answers.deploymentTarget === 'async-inference' || answers.deploymentTarget === 'batch-transform') {
+        ignorePatterns.push('**/do/ic/**');
+        ignorePatterns.push('**/do/add-ic');
+        ignorePatterns.push('**/do/status');
+    }
+
     // Resolve architecture
     const resolver = new DeploymentConfigResolver();
     let architecture = answers.architecture;
@@ -325,6 +341,13 @@ export async function writeProject(templateDir, destDir, answers, registryConfig
     // Exclude do/benchmark when benchmarking is not selected
     if (!answers.includeBenchmark) {
         ignorePatterns.push('**/do/benchmark');
+        ignorePatterns.push('**/do/optimize');
+    }
+
+    // Exclude do/adapter and do/adapters/ when LoRA is not enabled
+    if (!answers.enableLora) {
+        ignorePatterns.push('**/do/adapter');
+        ignorePatterns.push('**/do/adapters/**');
     }
 
     // Exclude do/test when hosted-model-endpoint is not selected
@@ -567,7 +590,11 @@ async function _ensureTemplateVariables(answers, registryConfigManager = null) {
         baseImage: null,
         modelSource: 'huggingface',
         artifactUri: '',
-        modelLoadStrategy: 'runtime'
+        modelLoadStrategy: 'runtime',
+        existingEndpointName: null,
+        enableLora: false,
+        maxLoras: 30,
+        maxLoraRank: 64
     };
 
     Object.entries(defaults).forEach(([key, value]) => {
@@ -1052,7 +1079,11 @@ function _setExecutablePermissions(destDir) {
         'do/register',
         'do/ci',
         'do/manifest',
-        'do/benchmark'
+        'do/benchmark',
+        'do/optimize',
+        'do/status',
+        'do/add-ic',
+        'do/adapter'
     ];
 
     shellScripts.forEach(script => {

package/src/lib/bootstrap-command-handler.js

@@ -199,6 +199,9 @@ export default class BootstrapCommandHandler {
             if (stackOutputs.BatchS3BucketName) {
                 profileData.batchS3Bucket = stackOutputs.BatchS3BucketName;
             }
+            if (stackOutputs.AdapterS3BucketName) {
+                profileData.adapterS3Bucket = stackOutputs.AdapterS3BucketName;
+            }
             if (stackOutputs.BenchmarkS3BucketName) {
                 profileData.benchmarkS3Bucket = stackOutputs.BenchmarkS3BucketName;
             }
@@ -390,6 +393,9 @@ export default class BootstrapCommandHandler {
                 if (outputs.BatchS3BucketName) {
                     console.log(`  ✅ S3 bucket (batch): ${outputs.BatchS3BucketName}`);
                 }
+                if (outputs.AdapterS3BucketName) {
+                    console.log(`  ✅ S3 bucket (adapters): ${outputs.AdapterS3BucketName}`);
+                }
                 if (outputs.BenchmarkS3BucketName) {
                     console.log(`  ✅ S3 bucket (benchmark): ${outputs.BenchmarkS3BucketName}`);
                 }

package/src/lib/cli-handler.js

@@ -204,7 +204,7 @@ VALIDATION OPTIONS:
 
 MCP OPTIONS:
   --smart                     Enable Bedrock-powered smart mode on all MCP servers
-  --discover                  Enable live registry lookups (e.g. Docker Hub) on MCP servers that support it
+  --no-discover               Disable live registry lookups (HuggingFace API, quota checks) — catalog-only mode
 
 REGISTRY SYSTEM:
   The generator includes built-in registries for frameworks, models, and instance types:

package/src/lib/config-manager.js

@@ -1056,6 +1056,39 @@ export default class ConfigManager {
                 required: false,
                 default: null,
                 valueSpace: 'bounded'
+            },
+            enableLora: {
+                cliOption: 'enable-lora',
+                envVar: null,
+                configFile: true,
+                packageJson: false,
+                mcp: false,
+                promptable: true,
+                required: false,
+                default: false,
+                valueSpace: 'bounded'
+            },
+            maxLoras: {
+                cliOption: 'max-loras',
+                envVar: null,
+                configFile: true,
+                packageJson: false,
+                mcp: false,
+                promptable: true,
+                required: false,
+                default: 30,
+                valueSpace: 'bounded'
+            },
+            maxLoraRank: {
+                cliOption: 'max-lora-rank',
+                envVar: null,
+                configFile: true,
+                packageJson: false,
+                mcp: false,
+                promptable: true,
+                required: false,
+                default: 64,
+                valueSpace: 'bounded'
             }
         };
     }
@@ -1088,7 +1121,7 @@ export default class ConfigManager {
      */
     _parseValue(parameter, value) {
         // Handle boolean parameters
-        if (parameter === 'includeSampleModel' || parameter === 'includeTesting' || parameter === 'skipPrompts' || parameter === 'includeBenchmark' || parameter === 'benchmarkStreaming') {
+        if (parameter === 'includeSampleModel' || parameter === 'includeTesting' || parameter === 'skipPrompts' || parameter === 'includeBenchmark' || parameter === 'benchmarkStreaming' || parameter === 'enableLora') {
             return value === true || value === 'true';
         }
         
@@ -1598,7 +1631,7 @@ export default class ConfigManager {
         if (!mcpServerConfigs || !mcpServerConfigs[serverName]) return null;
 
         const smart = this.options.smart === true;
-        const discover = this.options.discover === true;
+        const discover = this.options.discover !== false;
         const serverConfig = mcpServerConfigs[serverName];
 
         // Build a custom McpClient that passes context through
@@ -1924,6 +1957,12 @@ export default class ConfigManager {
                 if (param === 'instanceType' && finalConfig.deploymentTarget === 'hyperpod-eks' && !finalConfig.instanceType) {
                     return; // Skip validation only if truly missing for backward compat
                 }
+
+                // Special case: instanceType is not required when attaching to an existing endpoint
+                // The instance type is inherited from the existing endpoint configuration
+                if (param === 'instanceType' && finalConfig.existingEndpointName) {
+                    return; // Skip validation — instance is inherited from existing endpoint
+                }
                 
                 if (isEmpty) {
                     if (config.promptable) {

package/src/lib/deployment-entry-schema.js

@@ -57,6 +57,22 @@ export default {
                 },
                 buildTarget: {
                     type: ['string', 'null']
+                },
+                icList: {
+                    type: 'array',
+                    items: {
+                        type: 'object',
+                        required: ['name'],
+                        properties: {
+                            name: { type: 'string', minLength: 1 },
+                            image: { type: 'string' },
+                            gpuCount: { type: 'integer', minimum: 0 },
+                            copyCount: { type: 'integer', minimum: 1 },
+                            isAdapter: { type: 'boolean' },
+                            baseIcName: { type: 'string' },
+                            artifactUrl: { type: 'string' }
+                        }
+                    }
                 }
             }
         },

package/src/lib/mcp-client.js

@@ -32,7 +32,7 @@ class McpClient {
         this.timeout = options.timeout || DEFAULT_TIMEOUT;
         this.parameterMatrix = options.parameterMatrix || {};
         this.smart = options.smart || false;
-        this.discover = options.discover || false;
+        this.discover = options.discover !== undefined ? options.discover : true;
         this._transport = null;
         this._client = null;
         this._diagnosticMessage = null;
@@ -98,10 +98,10 @@ class McpClient {
 
         // Build environment: merge process.env with server-specific env
         // When --smart flag is active, inject BEDROCK_SMART=true for this run
-        // When --discover flag is active, inject MCP_DISCOVER=true for this run
+        // Discover mode is now default; inject DISCOVER_MODE=false only when explicitly disabled
         // Always pass process.env so child processes inherit AWS credentials, profiles, etc.
         const smartEnv = this.smart ? { BEDROCK_SMART: 'true' } : {};
-        const discoverEnv = this.discover ? { MCP_DISCOVER: 'true' } : {};
+        const discoverEnv = this.discover === false ? { DISCOVER_MODE: 'false' } : {};
         const serverEnv = env && Object.keys(env).length > 0 ? env : {};
         const spawnEnv = { ...process.env, ...smartEnv, ...discoverEnv, ...serverEnv };
 

package/src/lib/prompt-runner.js

@@ -18,8 +18,10 @@ import {
     modelLoadStrategyPrompts,
     modelProfilePrompts,
     modulePrompts,
+    loraPrompts,
     benchmarkPrompts,
     infraRegionAndTargetPrompts,
+    infraExistingEndpointPrompts,
     infraInstancePrompts,
     infraAsyncPrompts,
     infraBatchTransformPrompts,
@@ -29,7 +31,9 @@ import {
     destinationPrompts,
     baseImageSearchPrompts,
     baseImagePrompts,
-    formatImageChoices
+    formatImageChoices,
+    filterByCudaGeneration,
+    instanceCatalogRaw
 } from './prompts.js';
 
 import fs from 'fs';
@@ -187,12 +191,40 @@ export default class PromptRunner {
         // 3a. Region query
         await this._queryMcpForRegion(frameworkAnswers, explicitConfig);
 
+        // 3a2. Existing endpoint prompt (only for realtime-inference)
+        // Requirements: 3.3, 4.3, 4.4 — endpoint-picker MCP query
+        let existingEndpointAnswers = {};
+        if (regionAndTargetAnswers.deploymentTarget === 'realtime-inference') {
+            // Query endpoint-picker MCP server for available endpoints
+            const resolvedRegion = regionAndTargetAnswers.customAwsRegion || regionAndTargetAnswers.awsRegion;
+            await this._queryMcpForEndpoints({ ...regionAndTargetAnswers, awsRegion: resolvedRegion }, explicitConfig);
+
+            const endpointPreviousAnswers = {
+                ...regionAndTargetAnswers,
+                ...(this._mcpEndpointChoices ? { _mcpEndpointChoices: this._mcpEndpointChoices } : {})
+            };
+            existingEndpointAnswers = await this._runPhase(
+                infraExistingEndpointPrompts,
+                endpointPreviousAnswers,
+                explicitConfig,
+                existingConfig
+            );
+
+            // Resolve custom endpoint name
+            if (existingEndpointAnswers.customExistingEndpointName) {
+                existingEndpointAnswers.existingEndpointName = existingEndpointAnswers.customExistingEndpointName;
+                delete existingEndpointAnswers.customExistingEndpointName;
+            }
+        }
+
         // 3b. Instance type — query instance-sizer with full context (model + profile + CUDA)
         let instanceAnswers = {};
-        const needsInstance = regionAndTargetAnswers.deploymentTarget === 'realtime-inference' ||
+        // Skip instance prompts when attaching to an existing endpoint (instance is inherited)
+        const useExistingEndpoint = !!(existingEndpointAnswers.existingEndpointName);
+        const needsInstance = !useExistingEndpoint && (regionAndTargetAnswers.deploymentTarget === 'realtime-inference' ||
             regionAndTargetAnswers.deploymentTarget === 'async-inference' ||
             regionAndTargetAnswers.deploymentTarget === 'batch-transform' ||
-            regionAndTargetAnswers.deploymentTarget === 'hyperpod-eks';
+            regionAndTargetAnswers.deploymentTarget === 'hyperpod-eks');
 
         if (needsInstance) {
             // Determine architecture type for heuristic fallback
@@ -230,6 +262,74 @@ export default class PromptRunner {
             if (!instanceAnswers.instanceType && !explicitConfig.instanceType && this._architectureHeuristicDefault) {
                 instanceAnswers.instanceType = this._architectureHeuristicDefault;
             }
+
+            // Process multi-select instance type results (Requirements: 6.4)
+            // When user selects multiple instances via checkbox, derive instanceType and instancePools
+            if (instanceAnswers.instanceTypeSelections && instanceAnswers.instanceTypeSelections.length > 0) {
+                let selections = instanceAnswers.instanceTypeSelections.slice(0, 5); // Cap at 5 (API limit)
+
+                // Resolve custom input: replace __custom_input__ sentinel with parsed instances
+                if (selections.includes('__custom_input__') && instanceAnswers.customInstanceTypeSelections) {
+                    const customInstances = instanceAnswers.customInstanceTypeSelections
+                        .split(',').map(s => s.trim()).filter(s => s.length > 0);
+                    // Remove the sentinel and any other MCP selections, replace with custom entries
+                    selections = selections.filter(s => s !== '__custom_input__');
+                    selections = [...selections, ...customInstances];
+                    delete instanceAnswers.customInstanceTypeSelections;
+                } else if (selections.includes('__custom_input__')) {
+                    // Sentinel selected but no custom input provided — remove it
+                    selections = selections.filter(s => s !== '__custom_input__');
+                }
+
+                // Cap at 5 after custom expansion
+                if (selections.length > 5) {
+                    console.log('   ⚠️  Maximum 5 instance types allowed. Using first 5 selections.');
+                    selections = selections.slice(0, 5);
+                }
+
+                // Filter to same CUDA generation and warn about incompatible removals
+                const { filtered, generation, removed } = filterByCudaGeneration(selections);
+                if (removed.length > 0) {
+                    console.log(`   ⚠️  Removed incompatible instances (different CUDA generation): ${removed.join(', ')}`);
+                    console.log(`   Keeping ${generation} generation: ${filtered.join(', ')}`);
+                }
+
+                const finalSelections = filtered.length > 0 ? filtered : selections;
+
+                if (finalSelections.length === 1) {
+                    // Single selection → standard single instance type (no pools)
+                    instanceAnswers.instanceType = finalSelections[0];
+                    console.log(`   ✓ Single instance selected: ${finalSelections[0]}`);
+                } else {
+                    // Multiple selections → instance pools with priority = selection order
+                    instanceAnswers.instanceType = finalSelections[0]; // backward compat: first is primary
+                    instanceAnswers.instancePools = finalSelections.map((it, idx) => ({
+                        InstanceType: it,
+                        Priority: idx + 1
+                    }));
+
+                    // Auto-generate multi-spec IC config from catalog
+                    instanceAnswers.instancePoolSpecs = finalSelections.map(it => {
+                        const entry = instanceCatalogRaw[it];
+                        return {
+                            instanceType: it,
+                            gpuCount: entry?.gpus || 1,
+                            minMemoryMb: entry?.gpuMemoryGb ? entry.gpuMemoryGb * 1024 : 1024
+                        };
+                    });
+
+                    console.log(`   ✓ Instance pools configured (${finalSelections.length} types):`);
+                    finalSelections.forEach((it, idx) => {
+                        const entry = instanceCatalogRaw[it];
+                        const gpus = entry?.gpus || '?';
+                        const mem = entry?.gpuMemoryGb || '?';
+                        console.log(`     Priority ${idx + 1}: ${it} (${gpus} GPUs, ${mem}GB GPU memory)`);
+                    });
+                }
+
+                // Clean up the raw selections from answers (not needed downstream)
+                delete instanceAnswers.instanceTypeSelections;
+            }
         }
 
         // In auto-prompt mode, use instance-sizer's top recommendation as the instance type
@@ -318,6 +418,7 @@ export default class PromptRunner {
         // Combine all infrastructure answers
         const infraAnswers = {
             ...regionAndTargetAnswers,
+            ...existingEndpointAnswers,
             ...instanceAnswers,
             ...asyncAnswers,
             ...batchTransformAnswers,
@@ -414,6 +515,14 @@ export default class PromptRunner {
             }
         }
 
+        // LoRA adapter prompts — only for transformers with vllm/sglang/djl-lmi
+        // Requirements: 1.1, 1.2, 1.4
+        let loraAnswers = {};
+        const loraSubAnswers = await this._runPhase(loraPrompts, { ...frameworkAnswers, ...engineAnswers }, explicitConfig, existingConfig);
+        if (loraSubAnswers.enableLora !== undefined) {
+            loraAnswers = loraSubAnswers;
+        }
+
         // Validate instance type against framework requirements (now that framework version is known)
         const finalInstanceType = infraAnswers.customInstanceType || infraAnswers.instanceType;
         if (finalInstanceType && frameworkVersionAnswers.frameworkVersion) {
@@ -456,6 +565,7 @@ export default class PromptRunner {
             ...ngcApiKeyAnswers,
             ...moduleAnswers,
             ...benchmarkAnswers,
+            ...loraAnswers,
             ...projectAnswers,
             ...destinationAnswers,
             buildTimestamp
@@ -988,9 +1098,9 @@ export default class PromptRunner {
         if (!modelName || modelName === 'Custom (enter manually)') return;
 
         const smart = this.options.smart === true;
-        const discover = this.options.discover === true;
+        const discover = this.options.discover !== false;
 
-        const modeLabel = [smart && '[smart]', discover && '[discover]'].filter(Boolean).join(' ');
+        const modeLabel = [smart && '[smart]', !discover && '[no-discover]'].filter(Boolean).join(' ');
         console.log(`   🔍 Querying instance-sizer${modeLabel ? ` ${modeLabel}` : ''}...`);
 
         try {
@@ -1005,8 +1115,8 @@ export default class PromptRunner {
             const { StdioClientTransport } = await import('@modelcontextprotocol/sdk/client/stdio.js');
 
             const serverArgs = [...(serverConfig.args || [])];
-            if (discover && !serverArgs.includes('--discover')) {
-                serverArgs.push('--discover');
+            if (!discover && !serverArgs.includes('--no-discover')) {
+                serverArgs.push('--no-discover');
             }
 
             const transport = new StdioClientTransport({
@@ -1083,6 +1193,11 @@ export default class PromptRunner {
 
                     console.log(`   ✓ ${choices.length} compatible instance(s) found${vramInfo}`);
 
+                    // Warn if all instances had zero quota but were restored for visibility
+                    if (parsed.metadata?.allFilteredByQuota) {
+                        console.log('   ⚠️  All instances have zero quota — request a quota increase for your preferred type');
+                    }
+
                     // Check if availability data is present (recommendations have capacityType)
                     const hasAvailabilityData = recommendations.some(r => r.capacityType);
 
@@ -1187,6 +1302,62 @@ export default class PromptRunner {
         }
     }
 
+    /**
+     * Query the endpoint-picker MCP server for available InService real-time endpoints.
+     * Populates this._mcpEndpointChoices for the existing endpoint selection prompt.
+     * Graceful fallback: if MCP server fails (no credentials, timeout), skip and create new endpoint.
+     * Requirements: 3.3, 4.3, 4.4
+     * @private
+     */
+    async _queryMcpForEndpoints(infraAnswers, explicitConfig) {
+        const cm = this.configManager;
+        if (!cm) return;
+
+        const mcpServers = cm.getMcpServerNames();
+        if (!mcpServers.includes('endpoint-picker')) return;
+
+        // Skip if existing endpoint already provided via CLI/config
+        if (explicitConfig.existingEndpointName) return;
+
+        console.log('   🔍 Querying endpoint-picker...');
+
+        try {
+            const result = await cm.queryMcpServer('endpoint-picker', {
+                awsRegion: infraAnswers.awsRegion,
+                deploymentTarget: 'realtime-inference'
+            });
+
+            if (result && result.choices?.endpointName?.length > 0) {
+                const endpointNames = result.choices.endpointName;
+                const metadata = result.metadata || {};
+
+                // Build choices with metadata annotations
+                this._mcpEndpointChoices = endpointNames.map(name => {
+                    const meta = metadata[name];
+                    if (meta) {
+                        const gpuInfo = meta.availableGpus === '?' ? 'GPUs: ?' : `${meta.availableGpus} GPUs free`;
+                        return {
+                            name: `${name} (${meta.instanceType}, ${gpuInfo}, ${meta.icCount} IC${meta.icCount !== 1 ? 's' : ''})`,
+                            value: name
+                        };
+                    }
+                    return { name, value: name };
+                });
+
+                console.log(`   ✓ ${endpointNames.length} endpoint(s) with available capacity`);
+            } else {
+                if (result?.message) {
+                    console.log(`   ↳ ${result.message}`);
+                } else {
+                    console.log('   ↳ No endpoints with available capacity found');
+                }
+            }
+        } catch (err) {
+            // Graceful fallback: if MCP server fails, skip and create new endpoint
+            console.log(`   ⚠️  endpoint-picker: ${err.message || 'query failed'} — will create new endpoint`);
+        }
+    }
+
     /**
      * Query MCP base-image-picker server after deployment config is selected.
      * Populates _mcpBaseImageChoices for the base image selection prompt.
@@ -1204,7 +1375,7 @@ export default class PromptRunner {
         if (!mcpServers.includes('base-image-picker')) return;
 
         const smart = this.options.smart === true;
-        const discover = this.options.discover === true;
+        const discover = this.options.discover !== false;
         const framework = frameworkAnswers.framework;
         const modelServer = frameworkAnswers.modelServer;
         const architecture = frameworkAnswers.architecture || frameworkAnswers.deploymentConfig?.split('-')[0];