@aws/ml-container-creator 0.13.3 → 0.13.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aws/ml-container-creator",
-  "version": "0.13.3",
+  "version": "0.13.4",
   "description": "Build and deploy custom ML containers on AWS SageMaker with minimal configuration.",
   "main": "src/index.js",
   "bin": {
@@ -119,7 +119,7 @@
     "@aws-sdk/client-sagemaker": "^3.700.0",
     "@aws-sdk/client-service-quotas": "^3.700.0",
     "@microsoft/eslint-formatter-sarif": "^3.1.0",
-    "eslint": "^8.57.0",
+    "eslint": "^8.57.1",
     "eslint-plugin-property-test-rules": "file:eslint-rules",
     "fast-check": "^4.5.2",
     "husky": "^9.1.7",

package/servers/instance-sizer/lib/model-resolver.js

@@ -2,137 +2,124 @@
 // SPDX-License-Identifier: Apache-2.0
 
 /**
- * Model Metadata Resolver
+ * Model Metadata Resolver for the instance-sizer MCP server.
  *
- * Three-tier resolution strategy for model metadata:
- * 1. Check model-sizes catalog (exact match or glob pattern match)
- * 2. If discover mode enabled, fetch HuggingFace config.json
- * 3. If neither available, return null (caller handles fallback)
+ * Resolution pipeline:
+ *   1. Load model-sizes catalog (pre-built, offline-safe)
+ *   2. Attempt catalog lookup via glob pattern matching
+ *   3. If discover=true and no catalog hit, fetch config.json from HuggingFace Hub
+ *   4. Extract/estimate parameter count, dtype, architecture, context length
+ *
+ * Exports:
+ *   - resolveModelMetadata(modelName, options) — full resolution pipeline
+ *   - globMatch(pattern, string) — glob-style pattern matching (case-insensitive)
+ *   - loadCatalog(path) — loads the model-sizes JSON catalog
+ *   - catalogLookup(modelName, catalog) — finds a model in the catalog
+ *   - estimateParamsFromConfig(config) — estimates params from architecture dimensions
+ *   - extractFromHuggingFaceConfig(config) — extracts metadata from HF config.json
  */
 
-import { readFile } from 'node:fs/promises';
+import { readFileSync } from 'node:fs';
+import { resolve, dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { dirname, join } from 'node:path';
-
-// ── Constants ────────────────────────────────────────────────────────────────
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
-const DEFAULT_CATALOG_PATH = join(__dirname, '..', '..', 'lib', 'catalogs', 'models.json');
-const HUGGINGFACE_BASE_URL = 'https://huggingface.co';
-const HUGGINGFACE_TIMEOUT_MS = 5000;
+const DEFAULT_CATALOG_PATH = resolve(__dirname, '../../lib/catalogs/model-sizes.json');
 
-// ── Glob Pattern Matching ────────────────────────────────────────────────────
+/**
+ * Known protocol prefixes that indicate non-HuggingFace model sources.
+ * Models with these prefixes are never fetched from HuggingFace Hub.
+ */
+export const PROTOCOL_PREFIXES = [
+    's3://',
+    'registry://',
+    'marketplace://',
+    'jumpstart://',
+    'jumpstart-hub://'
+];
 
 /**
- * Simple glob pattern matcher supporting * wildcards.
- * Case-insensitive matching.
+ * Determine if a model name matches the HuggingFace pattern: org/model-name.
+ * Must contain exactly one `/` and must NOT start with a protocol prefix.
  *
- * @param {string} pattern - Glob pattern (e.g., 'meta-llama/Llama-2-7b*')
- * @param {string} text - Text to match against
- * @returns {boolean} Whether the text matches the pattern
+ * @param {string} modelName - The model identifier to test
+ * @returns {boolean} Whether the model name is a HuggingFace model ID
  */
-const globMatch = (pattern, text) => {
-    const regexStr = pattern
-        .replace(/[.+^${}()|[\]\\]/g, '\\$&')
-        .replace(/\*/g, '.*');
-    const regex = new RegExp(`^${regexStr}$`, 'i');
-    return regex.test(text);
-};
+export function isHuggingFacePattern(modelName) {
+    if (!modelName || typeof modelName !== 'string') {
+        return false;
+    }
+    if (PROTOCOL_PREFIXES.some(prefix => modelName.startsWith(prefix))) {
+        return false;
+    }
+    const slashCount = (modelName.match(/\//g) || []).length;
+    return slashCount === 1;
+}
 
-// ── Catalog Lookup ───────────────────────────────────────────────────────────
+/**
+ * Glob-style pattern matching (case-insensitive).
+ * Supports `*` as a wildcard that matches any sequence of characters.
+ *
+ * @param {string} pattern - Pattern with optional `*` wildcards
+ * @param {string} string - String to test against the pattern
+ * @returns {boolean} Whether the string matches the pattern
+ */
+export function globMatch(pattern, string) {
+    // Escape regex special characters except `*`
+    const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&');
+    // Replace `*` with `.*` for regex matching
+    const regex = new RegExp(`^${escaped.replace(/\*/g, '.*')}$`, 'i');
+    return regex.test(string);
+}
 
 /**
- * Load the model-sizes catalog from disk.
+ * Load the model-sizes catalog from a JSON file.
  *
- * @param {string} [catalogPath] - Path to catalog JSON file
- * @returns {Promise<object|null>} Parsed catalog or null on failure
+ * @param {string} [catalogPath] - Absolute path to the catalog JSON file
+ * @returns {Promise<Object|null>} Parsed catalog object, or null if file not found/invalid
  */
-const loadCatalog = async (catalogPath) => {
+export async function loadCatalog(catalogPath = DEFAULT_CATALOG_PATH) {
     try {
-        const raw = await readFile(catalogPath || DEFAULT_CATALOG_PATH, 'utf-8');
+        const raw = readFileSync(catalogPath, 'utf8');
         return JSON.parse(raw);
     } catch {
         return null;
     }
-};
+}
 
 /**
- * Look up a model in the catalog by exact match or glob pattern.
+ * Look up a model in the catalog using glob pattern matching.
+ * Iterates over catalog keys (which may contain `*` wildcards) and
+ * returns the first matching entry.
  *
- * @param {string} modelName - HuggingFace model ID or catalog key
- * @param {object} catalog - Parsed catalog object (flat or with .models wrapper)
- * @returns {object|null} Catalog entry or null if not found
+ * @param {string} modelName - HuggingFace model ID (e.g., "meta-llama/Llama-3.1-8B-Instruct")
+ * @param {Object|null} catalog - Loaded catalog object with a `models` field
+ * @returns {Object|null} Matching catalog entry, or null
  */
-const catalogLookup = (modelName, catalog) => {
-    if (!catalog) {
+export function catalogLookup(modelName, catalog) {
+    if (!catalog || !catalog.models) {
         return null;
     }
 
-    // Support both flat catalog (models.json) and wrapped format ({ models: {...} })
-    const models = catalog.models || catalog;
-
-    // Try exact match first
-    if (models[modelName]) {
-        return models[modelName];
-    }
-
-    // Try glob pattern matching
-    for (const pattern of Object.keys(models)) {
+    for (const [pattern, entry] of Object.entries(catalog.models)) {
         if (globMatch(pattern, modelName)) {
-            return models[pattern];
+            return entry;
         }
     }
 
     return null;
-};
-
-// ── HuggingFace API ──────────────────────────────────────────────────────────
+}
 
 /**
- * Fetch model config.json from HuggingFace Hub.
+ * Estimate parameter count from model architecture dimensions.
+ * Uses the approximation: hidden_size × num_hidden_layers × 12.
  *
- * @param {string} modelName - HuggingFace model ID (e.g., 'meta-llama/Llama-2-7b-chat-hf')
- * @returns {Promise<object|null>} Parsed config or null on failure
+ * @param {Object} config - Model configuration (HuggingFace config.json format)
+ * @returns {number|null} Estimated parameter count, or null if dimensions are missing
  */
-const fetchHuggingFaceConfig = async (modelName) => {
-    const url = `${HUGGINGFACE_BASE_URL}/${modelName}/resolve/main/config.json`;
-
-    try {
-        const controller = new AbortController();
-        const timeout = setTimeout(() => controller.abort(), HUGGINGFACE_TIMEOUT_MS);
-
-        const response = await fetch(url, {
-            signal: controller.signal,
-            headers: { 'Accept': 'application/json' }
-        });
-
-        clearTimeout(timeout);
-
-        if (!response.ok) {
-            return null;
-        }
-
-        return await response.json();
-    } catch {
-        return null;
-    }
-};
-
-/**
- * Estimate parameter count from architecture dimensions.
- * Uses the approximation: hidden_size × num_hidden_layers × 12
- *
- * This accounts for:
- * - Attention weights (Q, K, V, O projections = 4 × hidden_size²)
- * - FFN weights (typically 8 × hidden_size²)
- * - Embeddings and other components
- *
- * @param {object} config - HuggingFace config.json contents
- * @returns {number|null} Estimated parameter count or null if dimensions unavailable
- */
-const estimateParamsFromConfig = (config) => {
+export function estimateParamsFromConfig(config) {
     const hiddenSize = config.hidden_size;
     const numLayers = config.num_hidden_layers;
 
@@ -141,20 +128,18 @@ const estimateParamsFromConfig = (config) => {
     }
 
     return hiddenSize * numLayers * 12;
-};
+}
 
 /**
- * Extract model metadata from a HuggingFace config.json.
+ * Extract model metadata from a HuggingFace config.json object.
  *
- * @param {object} config - Parsed HuggingFace config.json
- * @returns {object} Extracted metadata
+ * @param {Object} config - Parsed config.json from HuggingFace Hub
+ * @returns {Object} Extracted metadata with parameterCount, dtype, architecture, maxPositionEmbeddings, source
  */
-const extractFromHuggingFaceConfig = (config) => {
-    const parameterCount = config.num_parameters
-        ?? estimateParamsFromConfig(config);
-
+export function extractFromHuggingFaceConfig(config) {
+    const parameterCount = config.num_parameters || estimateParamsFromConfig(config);
     const dtype = config.torch_dtype || 'float16';
-    const architecture = config.architectures?.[0] || 'unknown';
+    const architecture = (config.architectures && config.architectures[0]) || 'unknown';
     const maxPositionEmbeddings = config.max_position_embeddings || 4096;
 
     return {
@@ -164,106 +149,63 @@ const extractFromHuggingFaceConfig = (config) => {
         maxPositionEmbeddings,
         source: 'huggingface_api'
     };
-};
-
-// ── In-memory cache for discover mode ────────────────────────────────────────
-
-const discoverCache = new Map();
-
-// ── Protocol prefix detection ────────────────────────────────────────────────
-
-const PROTOCOL_PREFIXES = ['jumpstart://', 'jumpstart-hub://', 's3://', 'registry://'];
+}
 
 /**
- * Check if a model name matches the HuggingFace org/model-name pattern.
- * Must contain exactly one `/` and no protocol prefix.
+ * Resolve model metadata through the full pipeline:
+ *   1. Catalog lookup (offline, fast)
+ *   2. HuggingFace Hub fetch (if discover=true and no catalog hit)
  *
- * @param {string} modelName - Model identifier to check
- * @returns {boolean} True if it matches the HuggingFace pattern
+ * @param {string} modelName - HuggingFace model ID
+ * @param {Object} [options] - Resolution options
+ * @param {string} [options.catalogPath] - Path to model-sizes catalog
+ * @param {boolean} [options.discover=true] - Whether to fetch from HuggingFace Hub on cache miss
+ * @param {number} [options.timeout=5000] - HTTP timeout for HuggingFace API (ms)
+ * @returns {Promise<Object|null>} Resolved metadata, or null if unresolvable
  */
-const isHuggingFacePattern = (modelName) => {
-    if (!modelName || typeof modelName !== 'string') return false;
-    // Must not have a protocol prefix
-    if (PROTOCOL_PREFIXES.some(prefix => modelName.startsWith(prefix))) return false;
-    // Must contain exactly one `/` (org/model-name)
-    const slashCount = (modelName.match(/\//g) || []).length;
-    return slashCount === 1;
-};
-
-// ── Main Resolver ────────────────────────────────────────────────────────────
-
-/**
- * Resolve model metadata from available sources.
- *
- * Three-tier resolution:
- * 1. Check model-sizes catalog (exact match or pattern match)
- * 2. If discover mode enabled AND model matches HuggingFace pattern, fetch config.json
- * 3. If neither available, return null
- *
- * @param {string} modelName - HuggingFace model ID or catalog key
- * @param {object} [options={}]
- * @param {boolean} [options.discover=false] - Enable HuggingFace API lookups
- * @param {string} [options.catalogPath] - Path to model-sizes catalog (for testing)
- * @returns {Promise<{ parameterCount: number, dtype: string, architecture: string, maxPositionEmbeddings: number, source: string } | null>}
- */
-const resolveModelMetadata = async (modelName, options = {}) => {
-    const { discover = true, catalogPath } = options;
-
-    // Tier 1: Catalog lookup
+export async function resolveModelMetadata(modelName, options = {}) {
+    const {
+        catalogPath = DEFAULT_CATALOG_PATH,
+        discover = true,
+        timeout = 5000
+    } = options;
+
+    // Step 1: Try catalog lookup
     const catalog = await loadCatalog(catalogPath);
     const catalogEntry = catalogLookup(modelName, catalog);
 
     if (catalogEntry) {
-        // Only use catalog entry if it has a usable parameterCount for VRAM estimation.
-        // If parameterCount is missing, fall through to HuggingFace API (tier 2).
-        if (catalogEntry.parameterCount) {
-            return {
-                parameterCount: catalogEntry.parameterCount,
-                dtype: catalogEntry.defaultDtype,
-                architecture: catalogEntry.architecture,
-                maxPositionEmbeddings: catalogEntry.maxPositionEmbeddings,
-                source: 'catalog'
-            };
-        }
+        return {
+            parameterCount: catalogEntry.parameterCount,
+            dtype: catalogEntry.defaultDtype || 'float16',
+            architecture: catalogEntry.architecture || 'unknown',
+            maxPositionEmbeddings: catalogEntry.maxPositionEmbeddings || 4096,
+            source: 'catalog'
+        };
     }
 
-    // Tier 2: HuggingFace API (only in discover mode, only for org/model-name pattern)
+    // Step 2: If discover mode, try HuggingFace Hub
     if (discover && isHuggingFacePattern(modelName)) {
-        // Check in-memory cache first
-        if (discoverCache.has(modelName)) {
-            return discoverCache.get(modelName);
-        }
+        try {
+            const controller = new AbortController();
+            const timer = setTimeout(() => controller.abort(), timeout);
 
-        const config = await fetchHuggingFaceConfig(modelName);
+            const url = `https://huggingface.co/${modelName}/resolve/main/config.json`;
+            const response = await fetch(url, {
+                signal: controller.signal,
+                headers: { 'User-Agent': 'ml-container-creator/instance-sizer' }
+            });
 
-        if (config) {
-            const metadata = extractFromHuggingFaceConfig(config);
+            clearTimeout(timer);
 
-            // Only return if we got a usable parameter count
-            if (metadata.parameterCount) {
-                // Cache for session duration
-                discoverCache.set(modelName, metadata);
-                return metadata;
+            if (response.ok) {
+                const config = await response.json();
+                return extractFromHuggingFaceConfig(config);
             }
+        } catch {
+            // Network error or timeout — fall through to null
         }
     }
 
-    // Tier 3: No metadata available
     return null;
-};
-
-export {
-    resolveModelMetadata,
-    globMatch,
-    loadCatalog,
-    catalogLookup,
-    fetchHuggingFaceConfig,
-    estimateParamsFromConfig,
-    extractFromHuggingFaceConfig,
-    isHuggingFacePattern,
-    discoverCache,
-    PROTOCOL_PREFIXES,
-    DEFAULT_CATALOG_PATH,
-    HUGGINGFACE_BASE_URL,
-    HUGGINGFACE_TIMEOUT_MS
-};
+}

package/servers/instance-sizer/lib/vram-estimator.js

@@ -31,6 +31,17 @@ const BYTES_IN_GB = 1024 ** 3;
 const DEFAULT_MAX_SEQUENCE_LENGTH = 4096;
 const DEFAULT_BATCH_SIZE = 1;
 const OVERHEAD_FACTOR = 0.1;
+const MIN_USEFUL_CONTEXT_LEN = 2048;
+
+const KV_CACHE_DTYPE_BYTES = {
+    'auto': 2,      // fp16/bf16
+    'fp16': 2,
+    'bfloat16': 2,
+    'fp8': 1,
+    'fp8_e5m2': 1,
+    'fp8_e4m3': 1,
+    'int8': 1
+};
 
 // ── Helper Functions ─────────────────────────────────────────────────────────
 
@@ -132,6 +143,78 @@ const estimateVram = (modelInfo) => {
     };
 };
 
+// ── Max Model Length Computation ─────────────────────────────────────────────
+
+/**
+ * Compute the maximum model length (context window) that fits in available KV cache memory.
+ *
+ * KV cache per token = 2 × num_layers × num_kv_heads × head_dim × dtype_bytes
+ * Available KV memory = (gpu_memory × gpu_count × utilization) - model_weight_memory - overhead
+ * max_model_len = available_kv_memory / kv_per_token
+ *
+ * For GQA models (like Llama 3.2), num_kv_heads is the GQA head count, not the
+ * full attention head count.
+ *
+ * @param {object} params
+ * @param {number} params.modelWeightGb - Model weight memory in GB
+ * @param {number} params.totalGpuMemoryGb - Total GPU memory for the instance
+ * @param {number} params.gpuCount - Number of GPUs
+ * @param {number} [params.gpuMemoryUtilization=0.9] - vLLM gpu_memory_utilization
+ * @param {number} params.numLayers - Number of transformer layers
+ * @param {number} params.numKvHeads - Number of KV attention heads (after GQA)
+ * @param {number} params.headDim - Dimension per head
+ * @param {string} [params.kvCacheDtype='auto'] - KV cache dtype
+ * @param {number} [params.overheadFactor=0.15] - Fraction for activations/CUDA overhead (conservative)
+ * @returns {{ maxModelLen: number, availableForKvGb: number } | null}
+ */
+const computeMaxModelLen = (params) => {
+    const {
+        modelWeightGb,
+        totalGpuMemoryGb,
+        gpuCount = 1,
+        gpuMemoryUtilization = 0.9,
+        numLayers,
+        numKvHeads,
+        headDim,
+        kvCacheDtype = 'auto',
+        overheadFactor = 0.15
+    } = params;
+
+    // Require architecture params — graceful degradation if missing
+    if (!numLayers || !numKvHeads || !headDim) {
+        return null;
+    }
+    if (!totalGpuMemoryGb || !modelWeightGb) {
+        return null;
+    }
+
+    // Compute available memory for KV cache
+    // Conservative: gpu_memory_utilization × (1 - overhead) gives usable memory
+    const usableMemoryGb = totalGpuMemoryGb * gpuCount * gpuMemoryUtilization * (1 - overheadFactor);
+    const availableForKvGb = usableMemoryGb - modelWeightGb;
+
+    if (availableForKvGb <= 0) {
+        return { maxModelLen: 0, availableForKvGb: 0 };
+    }
+
+    // KV cache per token (bytes)
+    // Formula: 2 (K+V) × num_layers × num_kv_heads × head_dim × dtype_bytes
+    // For TP: heads are distributed but so is memory — net effect is the same per-GPU
+    const dtypeBytes = KV_CACHE_DTYPE_BYTES[kvCacheDtype] || 2;
+    const kvPerTokenBytes = 2 * numLayers * numKvHeads * headDim * dtypeBytes;
+
+    // Convert available GB to bytes and divide
+    const availableBytes = availableForKvGb * BYTES_IN_GB;
+    const maxModelLen = Math.floor(availableBytes / kvPerTokenBytes);
+
+    return {
+        maxModelLen: Math.max(0, maxModelLen),
+        availableForKvGb
+    };
+};
+
+// ── Confidence ───────────────────────────────────────────────────────────────
+
 /**
  * Determine confidence level based on which parameters were explicitly provided.
  *
@@ -165,13 +248,16 @@ const determineConfidence = (modelInfo) => {
 
 export {
     estimateVram,
+    computeMaxModelLen,
     bytesPerParam,
     estimateKvCache,
     determineConfidence,
     BYTES_PER_PARAM,
     QUANTIZATION_BYTES,
+    KV_CACHE_DTYPE_BYTES,
     DEFAULT_MAX_SEQUENCE_LENGTH,
     DEFAULT_BATCH_SIZE,
     OVERHEAD_FACTOR,
+    MIN_USEFUL_CONTEXT_LEN,
     BYTES_IN_GB
 };

package/servers/lib/catalogs/instances.json

@@ -228,33 +228,6 @@
             "gpuMemoryGb": 24,
             "gpuType": "NVIDIA A10G",
             "costTier": "medium"
-        },
-        "ml.p6-b200.48xlarge": {
-            "category": "gpu",
-            "gpus": 8,
-            "vcpus": 192,
-            "memGb": 1536,
-            "accelerator": "8x B200 1440GB",
-            "cudaVersions": [
-                "12.4",
-                "12.6"
-            ],
-            "tags": [
-                "gpu",
-                "multi-gpu",
-                "b200",
-                "cuda-12",
-                "high-performance"
-            ],
-            "family": "p6",
-            "acceleratorType": "cuda",
-            "hardware": "NVIDIA B200",
-            "gpuArchitecture": "Blackwell",
-            "defaultCudaVersion": "12.6",
-            "notes": "8x NVIDIA B200 GPUs (1440GB total). Next-gen Blackwell architecture",
-            "gpuMemoryGb": 180,
-            "gpuType": "NVIDIA B200",
-            "costTier": "high"
         }
     },
     "recommendations": {

package/src/lib/bootstrap-command-handler.js

@@ -170,7 +170,7 @@ export default class BootstrapCommandHandler {
         console.log('\n🚀 Bootstrap — Shared AWS Infrastructure Setup\n');
 
         // Verify AWS CLI v2 is installed
-        if (!this.provisioners._verifyCliV2()) {
+        if (!this._verifyCliV2()) {
             return;
         }
 
@@ -261,7 +261,7 @@ export default class BootstrapCommandHandler {
                 }
 
                 profileData.roleArn = stackOutputs.RoleArn;
-                profileData.ecrRepositoryName = stackOutputs.EcrRepositoryName;
+                profileData.ecrRepositoryName = stackOutputs.EcrRepositoryName || 'ml-container-creator';
                 profileData.stackName = stackName;
                 profileData.sharedInfraFrom = otherStack;  // Track that this profile reuses another's stack
                 if (stackOutputs.AsyncS3BucketName) profileData.asyncS3Bucket = stackOutputs.AsyncS3BucketName;

package/src/lib/generated/cli-options.js

@@ -1,6 +1,6 @@
 // AUTO-GENERATED by scripts/codegen-cli.js — DO NOT EDIT
 // Source: config/parameter-schema-v2.json
-// Generated: 2026-06-12T22:03:00.429Z
+// Generated: 2026-06-15T20:16:03.840Z
 
 /**
  * CLI option definitions derived from parameter-schema-v2.json.

package/src/lib/generated/parameter-matrix.js

@@ -1,6 +1,6 @@
 // AUTO-GENERATED by scripts/codegen-parameter-matrix.js — DO NOT EDIT
 // Source: config/parameter-schema-v2.json
-// Generated: 2026-06-12T22:03:00.552Z
+// Generated: 2026-06-15T20:16:03.952Z
 
 /**
  * Parameter matrix defining how each parameter is loaded from various sources.

package/src/lib/generated/validation-rules.js

@@ -1,6 +1,6 @@
 // AUTO-GENERATED by scripts/codegen-validator.js — DO NOT EDIT
 // Source: config/parameter-schema-v2.json
-// Generated: 2026-06-12T22:03:00.468Z
+// Generated: 2026-06-15T20:16:03.877Z
 
 /**
  * Validation rules derived from parameter-schema-v2.json.