@aws/ml-container-creator 0.2.4 → 0.2.6

package/src/lib/prompts.js

@@ -14,7 +14,7 @@ import { fileURLToPath } from 'node:url';
 
 const __promptsFilename = fileURLToPath(import.meta.url);
 const __promptsDir = dirname(__promptsFilename);
-const instancesCatalogPath = resolve(__promptsDir, '../../servers/instance-recommender/catalogs/instances.json');
+const instancesCatalogPath = resolve(__promptsDir, '../../servers/lib/catalogs/instances.json');
 
 /**
  * Load instance types from the instances.json catalog and transform

package/src/lib/registry-loader.js

@@ -18,11 +18,11 @@ const __dirname = dirname(__filename);
 
 // Catalog file paths relative to this module
 const CATALOG_PATHS = {
-    modelServers: resolve(__dirname, '../../servers/base-image-picker/catalogs/model-servers.json'),
-    tritonBackends: resolve(__dirname, '../../servers/base-image-picker/catalogs/triton-backends.json'),
-    instances: resolve(__dirname, '../../servers/instance-recommender/catalogs/instances.json'),
-    popularTransformers: resolve(__dirname, '../../servers/model-picker/catalogs/popular-transformers.json'),
-    popularDiffusors: resolve(__dirname, '../../servers/model-picker/catalogs/popular-diffusors.json')
+    modelServers: resolve(__dirname, '../../servers/lib/catalogs/model-servers.json'),
+    tritonBackends: resolve(__dirname, '../../servers/lib/catalogs/triton-backends.json'),
+    instances: resolve(__dirname, '../../servers/lib/catalogs/instances.json'),
+    popularTransformers: resolve(__dirname, '../../servers/lib/catalogs/popular-transformers.json'),
+    popularDiffusors: resolve(__dirname, '../../servers/lib/catalogs/popular-diffusors.json')
 };
 
 class RegistryLoader {

package/src/lib/schema-sync.js

@@ -0,0 +1,203 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Schema Sync — Downloads AWS service model files from the AWS SDK GitHub source
+ * and stores them in the local schema registry.
+ *
+ * Requirements: 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 10.1
+ */
+
+import https from 'node:https';
+import { existsSync, mkdirSync, writeFileSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+const SERVICES = ['sagemaker', 'iam', 'ecr', 's3'];
+
+const SOURCE_BASE_URL = 'https://raw.githubusercontent.com/aws/aws-sdk-js-v3/main/codegen/sdk-codegen/aws-models';
+
+/**
+ * Get the default schema registry path.
+ * @returns {string}
+ */
+export function getRegistryPath() {
+    return path.join(os.homedir(), '.ml-container-creator', 'schemas');
+}
+
+/**
+ * Download a file from a URL using the built-in https module.
+ * @param {string} url - URL to download
+ * @returns {Promise<string>} - Response body as string
+ */
+function downloadFile(url) {
+    return new Promise((resolve, reject) => {
+        https.get(url, (res) => {
+            if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+                downloadFile(res.headers.location).then(resolve).catch(reject);
+                return;
+            }
+
+            if (res.statusCode !== 200) {
+                reject(new Error(`HTTP ${res.statusCode} for ${url}`));
+                return;
+            }
+
+            const chunks = [];
+            res.on('data', (chunk) => chunks.push(chunk));
+            res.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
+            res.on('error', reject);
+        }).on('error', reject);
+    });
+}
+
+/**
+ * Count shapes and enums in a service model.
+ * @param {object} model - Parsed service model JSON
+ * @returns {{ shapeCount: number, enumCount: number, version: string }}
+ */
+function getModelStats(model) {
+    const shapes = model.shapes || {};
+    let shapeCount = 0;
+    let enumCount = 0;
+
+    for (const shape of Object.values(shapes)) {
+        shapeCount++;
+        if (shape.type === 'string' && shape.enum) {
+            enumCount++;
+        }
+        // Smithy models use traits for enums
+        if (shape.traits && shape.traits['smithy.api#enum']) {
+            enumCount++;
+        }
+        // Smithy v2 enum shapes
+        if (shape.type === 'enum') {
+            enumCount++;
+        }
+    }
+
+    const version = model.metadata?.apiVersion || '';
+
+    return { shapeCount, enumCount, version };
+}
+
+/**
+ * Sync all service models from the AWS SDK GitHub source.
+ * @param {object} [options]
+ * @param {string} [options.registryPath] - Override registry path
+ * @param {function} [options.downloadFn] - Override download function (for testing)
+ * @returns {Promise<{ success: boolean, services: object }>}
+ */
+export async function syncSchemas(options = {}) {
+    const registryPath = options.registryPath || getRegistryPath();
+    const download = options.downloadFn || downloadFile;
+
+    // Ensure registry directory exists
+    mkdirSync(registryPath, { recursive: true });
+
+    const serviceStats = {};
+    let hasErrors = false;
+
+    for (const service of SERVICES) {
+        const url = `${SOURCE_BASE_URL}/${service}.json`;
+        const serviceDir = path.join(registryPath, service);
+
+        try {
+            console.log(`  Syncing ${service}...`);
+            const content = await download(url);
+
+            // Parse to extract stats
+            let model;
+            try {
+                model = JSON.parse(content);
+            } catch (parseErr) {
+                console.log(`  ⚠️  ${service}: Failed to parse model — ${parseErr.message}`);
+                hasErrors = true;
+                continue;
+            }
+
+            const stats = getModelStats(model);
+
+            // Store the file
+            mkdirSync(serviceDir, { recursive: true });
+            writeFileSync(path.join(serviceDir, 'service-2.json'), content, 'utf8');
+
+            serviceStats[service] = {
+                shapeCount: stats.shapeCount,
+                enumCount: stats.enumCount,
+                version: stats.version
+            };
+
+            console.log(`  ✅ ${service}: ${stats.shapeCount} shapes, ${stats.enumCount} enums`);
+        } catch (err) {
+            console.log(`  ⚠️  ${service}: ${err.message}`);
+            hasErrors = true;
+        }
+    }
+
+    // Write manifest
+    const manifest = {
+        lastSynced: new Date().toISOString(),
+        services: serviceStats,
+        source: 'https://github.com/aws/aws-sdk-js-v3/tree/main/codegen/sdk-codegen/aws-models'
+    };
+
+    writeFileSync(
+        path.join(registryPath, 'manifest.json'),
+        JSON.stringify(manifest, null, 4),
+        'utf8'
+    );
+
+    return { success: !hasErrors, services: serviceStats, manifest };
+}
+
+/**
+ * Load the manifest from the schema registry.
+ * @param {string} [registryPath] - Override registry path
+ * @returns {object|null} Parsed manifest or null if not found
+ */
+export function loadManifest(registryPath) {
+    const regPath = registryPath || getRegistryPath();
+    const manifestPath = path.join(regPath, 'manifest.json');
+
+    if (!existsSync(manifestPath)) {
+        return null;
+    }
+
+    try {
+        return JSON.parse(readFileSync(manifestPath, 'utf8'));
+    } catch {
+        return null;
+    }
+}
+
+/**
+ * Load a service model from the registry.
+ * @param {string} serviceName - Service name (e.g., 'sagemaker')
+ * @param {string} [registryPath] - Override registry path
+ * @returns {string|null} Raw file content or null if not found
+ */
+export function loadServiceModel(serviceName, registryPath) {
+    const regPath = registryPath || getRegistryPath();
+    const modelPath = path.join(regPath, serviceName, 'service-2.json');
+
+    if (!existsSync(modelPath)) {
+        return null;
+    }
+
+    return readFileSync(modelPath, 'utf8');
+}
+
+/**
+ * Store a service model in the registry.
+ * @param {string} serviceName - Service name (e.g., 'sagemaker')
+ * @param {string} content - Raw file content to store
+ * @param {string} [registryPath] - Override registry path
+ */
+export function storeServiceModel(serviceName, content, registryPath) {
+    const regPath = registryPath || getRegistryPath();
+    const serviceDir = path.join(regPath, serviceName);
+
+    mkdirSync(serviceDir, { recursive: true });
+    writeFileSync(path.join(serviceDir, 'service-2.json'), content, 'utf8');
+}

package/src/lib/schema-validation-engine.js

@@ -0,0 +1,195 @@
+import ValidationReport from './validation-report.js';
+import EnumValidator from './validators/enum-validator.js';
+import TypeValidator from './validators/type-validator.js';
+import RequiredFieldValidator from './validators/required-field-validator.js';
+import CrossCuttingChecker from './cross-cutting-checker.js';
+import { existsSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+
+/**
+ * Core validation orchestrator.
+ * Loads service models, runs static and smart validators, and produces a report.
+ *
+ * Requirements: 12.1, 12.2, 12.5, 15.4, 15.5
+ */
+export default class SchemaValidationEngine {
+    /**
+     * @param {Object} options
+     * @param {string} options.registryPath - Path to schema registry
+     * @param {boolean} options.ignoreStaleness - Suppress staleness warnings
+     * @param {boolean} options.smartMode - Enable smart-mode validators
+     */
+    constructor(options = {}) {
+        this.registryPath = options.registryPath || null;
+        this.ignoreStaleness = options.ignoreStaleness || false;
+        this.smartMode = options.smartMode || false;
+        this.validators = [];
+        this.serviceModels = options.serviceModels || [];
+        this.instanceCatalog = options.instanceCatalog || null;
+        this.crossCuttingChecker = new CrossCuttingChecker();
+
+        // Auto-register built-in validators
+        this.registerValidator(new EnumValidator());
+        this.registerValidator(new TypeValidator());
+        this.registerValidator(new RequiredFieldValidator());
+    }
+
+    /**
+     * Run full validation pipeline.
+     * Orchestrate: load models → run static validators → run smart validators (if enabled) → return report.
+     * @param {Object} context - ValidationContext from PayloadBuilder
+     * @returns {Promise<ValidationReport>}
+     */
+    async validate(context) {
+        const report = new ValidationReport();
+
+        // Run static validators (mode === 'static' or 'both')
+        const staticValidators = this.validators.filter(
+            v => v.mode === 'static' || v.mode === 'both'
+        );
+
+        const priorFindings = [];
+
+        for (const validator of staticValidators) {
+            try {
+                const findings = await validator.validate(context, {
+                    priorFindings: [...priorFindings],
+                    serviceModels: this.serviceModels
+                });
+                for (const finding of findings) {
+                    report.addFinding(finding);
+                    priorFindings.push(finding);
+                }
+            } catch (err) {
+                report.warnings.push({
+                    source: 'engine',
+                    severity: 'warning',
+                    operation: '',
+                    fieldPath: '',
+                    remediationHint: `Plugin "${validator.name}" threw an error: ${err.message}`
+                });
+            }
+        }
+
+        // Run cross-cutting checks after schema validators
+        if (this.instanceCatalog) {
+            try {
+                const crossCuttingFindings = this.crossCuttingChecker.check(context, this.instanceCatalog);
+                for (const finding of crossCuttingFindings) {
+                    report.addFinding(finding);
+                    priorFindings.push(finding);
+                }
+            } catch (err) {
+                report.warnings.push({
+                    source: 'engine',
+                    severity: 'warning',
+                    operation: '',
+                    fieldPath: '',
+                    remediationHint: `Cross-cutting checker threw an error: ${err.message}`
+                });
+            }
+        }
+
+        // Run smart validators if enabled (mode === 'smart' or 'both')
+        if (this.smartMode) {
+            const smartValidators = this.validators.filter(
+                v => v.mode === 'smart' || v.mode === 'both'
+            );
+
+            for (const validator of smartValidators) {
+                // Skip validators already run in static pass (mode === 'both')
+                if (validator.mode === 'both' && staticValidators.includes(validator)) {
+                    continue;
+                }
+
+                try {
+                    const findings = await validator.validate(context, {
+                        priorFindings: [...priorFindings],
+                        serviceModels: this.serviceModels
+                    });
+                    for (const finding of findings) {
+                        report.addFinding(finding);
+                        priorFindings.push(finding);
+                    }
+                } catch (err) {
+                    report.warnings.push({
+                        source: 'engine',
+                        severity: 'warning',
+                        operation: '',
+                        fieldPath: '',
+                        remediationHint: `Smart plugin "${validator.name}" threw an error: ${err.message}`
+                    });
+                }
+            }
+        }
+
+        return report;
+    }
+
+    /**
+     * Register a custom validator plugin.
+     * @param {Object} validator - A BaseValidator instance
+     */
+    registerValidator(validator) {
+        this.validators.push(validator);
+    }
+
+    /**
+     * Check schema registry staleness.
+     * @returns {{ stale: boolean, lastSynced: string|null, daysSinceSync: number, registryMissing?: boolean }}
+     */
+    checkStaleness() {
+        if (!this.registryPath) {
+            return {
+                stale: false,
+                lastSynced: null,
+                daysSinceSync: 0,
+                registryMissing: true
+            };
+        }
+
+        let manifest;
+        try {
+            const manifestPath = path.join(this.registryPath, 'manifest.json');
+
+            if (!existsSync(manifestPath)) {
+                return {
+                    stale: false,
+                    lastSynced: null,
+                    daysSinceSync: 0,
+                    registryMissing: true
+                };
+            }
+
+            manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
+        } catch {
+            return {
+                stale: false,
+                lastSynced: null,
+                daysSinceSync: 0,
+                registryMissing: true
+            };
+        }
+
+        if (!manifest || !manifest.lastSynced) {
+            return {
+                stale: false,
+                lastSynced: null,
+                daysSinceSync: 0,
+                registryMissing: true
+            };
+        }
+
+        const lastSynced = manifest.lastSynced;
+        const syncDate = new Date(lastSynced);
+        const now = new Date();
+        const daysSinceSync = Math.floor((now - syncDate) / (1000 * 60 * 60 * 24));
+        const stale = daysSinceSync > 30;
+
+        if (stale && !this.ignoreStaleness) {
+            console.log(`⚠️  Schema registry is ${daysSinceSync} days old. Run \`ml-container-creator bootstrap sync-schemas\` to update.`);
+        }
+
+        return { stale, lastSynced, daysSinceSync };
+    }
+}

package/src/lib/service-model-parser.js

@@ -0,0 +1,102 @@
+/* eslint-disable eqeqeq */
+/**
+ * Parses AWS service-2.json files into a queryable in-memory index.
+ * Extracts operations, shapes, enums, constraints, and metadata.
+ *
+ * Requirements: 13.1, 13.2, 13.3, 13.4, 13.5
+ */
+export default class ServiceModelParser {
+    /**
+     * Parse a service-2.json file into an indexed representation.
+     * @param {Object} rawModel - Parsed JSON content of service-2.json
+     * @returns {Object} ServiceModelIndex with metadata, operations (Map), and shapes (Map)
+     */
+    parse(rawModel) {
+        const metadata = rawModel.metadata || {};
+        const operations = new Map();
+        const shapes = new Map();
+
+        if (rawModel.operations) {
+            for (const [name, op] of Object.entries(rawModel.operations)) {
+                operations.set(name, {
+                    input: op.input ? op.input.shape : null,
+                    output: op.output ? op.output.shape : null,
+                    errors: (op.errors || []).map(e => e.shape)
+                });
+            }
+        }
+
+        if (rawModel.shapes) {
+            for (const [name, shape] of Object.entries(rawModel.shapes)) {
+                shapes.set(name, {
+                    type: shape.type,
+                    required: shape.required || [],
+                    members: shape.members ? new Map(Object.entries(shape.members)) : new Map(),
+                    enum: shape.enum || null,
+                    min: shape.min != null ? shape.min : null,
+                    max: shape.max != null ? shape.max : null,
+                    pattern: shape.pattern || null,
+                    member: shape.member || null,
+                    key: shape.key || null,
+                    value: shape.value || null
+                });
+            }
+        }
+
+        return { metadata, operations, shapes };
+    }
+
+    /**
+     * Get the input shape for an API operation.
+     * @param {Object} index - Parsed model index (ServiceModelIndex)
+     * @param {string} operationName - e.g., 'CreateEndpointConfig'
+     * @returns {Object|null} The resolved input shape definition, or null if not found
+     */
+    getOperationInputShape(index, operationName) {
+        const op = index.operations.get(operationName);
+        if (!op || !op.input) return null;
+        return this.resolveShape(index, op.input);
+    }
+
+    /**
+     * Resolve a shape reference to its full definition.
+     * @param {Object} index - Parsed model index
+     * @param {string} shapeName - Shape name from the model
+     * @returns {Object|null} ShapeDefinition or null if not found
+     */
+    resolveShape(index, shapeName) {
+        return index.shapes.get(shapeName) || null;
+    }
+
+    /**
+     * Extract all enum values for a given shape.
+     * @param {Object} index - Parsed model index
+     * @param {string} shapeName - Shape name
+     * @returns {Array<string>|null} Enum values or null if not an enum shape
+     */
+    getEnumValues(index, shapeName) {
+        const shape = index.shapes.get(shapeName);
+        if (!shape) return null;
+        return shape.enum || null;
+    }
+
+    /**
+     * Get statistics about the parsed model.
+     * @param {Object} index - Parsed model index
+     * @returns {{ shapeCount: number, enumCount: number, operationCount: number }}
+     */
+    getStats(index) {
+        let enumCount = 0;
+        for (const [, shape] of index.shapes) {
+            if (shape.enum && shape.enum.length > 0) {
+                enumCount++;
+            }
+        }
+
+        return {
+            shapeCount: index.shapes.size,
+            enumCount,
+            operationCount: index.operations.size
+        };
+    }
+}

package/src/lib/validate-runner.js

@@ -0,0 +1,167 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Validation runner — the Node.js module invoked by do/validate.
+ * Loads config, constructs payloads, runs SchemaValidationEngine, and prints a report.
+ *
+ * - Prints structured report (text format by default)
+ * - Exits with code 1 if errors found, 0 if clean
+ * - Includes service model version date and fields validated count on success
+ * - Supports --smart flag for future MCP validator integration
+ *
+ * Requirements: 9.2, 9.3, 9.4
+ */
+
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import path from 'node:path';
+import PayloadBuilder from './payload-builder.js';
+import SchemaValidationEngine from './schema-validation-engine.js';
+import ServiceModelParser from './service-model-parser.js';
+import { getRegistryPath, loadManifest } from './schema-sync.js';
+
+/**
+ * Parse a do/config shell file into a key-value object.
+ * Extracts lines matching: export KEY="value" or export KEY=value
+ *
+ * @param {string} configPath - Path to the do/config file
+ * @returns {Object} Parsed configuration values
+ */
+export function parseDoConfig(configPath) {
+    if (!existsSync(configPath)) {
+        return null;
+    }
+
+    const content = readFileSync(configPath, 'utf8');
+    const config = {};
+
+    for (const line of content.split('\n')) {
+        const match = line.match(/^export\s+([A-Z_][A-Z0-9_]*)=["']?([^"'\n]*)["']?/);
+        if (match) {
+            const [, key, value] = match;
+            config[key] = value;
+        }
+    }
+
+    return config;
+}
+
+/**
+ * Run the full validation pipeline.
+ *
+ * @param {Object} options
+ * @param {string} [options.configDir] - Path to the do/ directory containing config
+ * @param {string} [options.format] - Output format: 'text' (default) or 'json'
+ * @param {boolean} [options.smart] - Enable smart-mode validators
+ * @param {string} [options.registryPath] - Override schema registry path
+ * @param {Object} [options.config] - Pre-parsed config (overrides configDir loading)
+ * @returns {Promise<number>} Exit code (0 = pass, 1 = fail, 2 = cannot run)
+ */
+export async function run(options = {}) {
+    const format = options.format || 'text';
+    const smart = options.smart || false;
+    const registryPath = options.registryPath || getRegistryPath();
+
+    // Check schema registry exists
+    if (!existsSync(registryPath) || !existsSync(path.join(registryPath, 'manifest.json'))) {
+        console.log('⚠️  Schema registry not found.');
+        console.log('   Run: ml-container-creator bootstrap sync-schemas');
+        process.exit(2);
+        return 2;
+    }
+
+    // Load config
+    let config = options.config;
+    if (!config && options.configDir) {
+        const configPath = path.join(options.configDir, 'config');
+        config = parseDoConfig(configPath);
+        if (!config) {
+            console.log('❌ Could not load do/config');
+            process.exit(2);
+            return 2;
+        }
+    }
+
+    if (!config) {
+        console.log('❌ No configuration provided');
+        process.exit(2);
+        return 2;
+    }
+
+    const deploymentTarget = config.DEPLOYMENT_TARGET || 'realtime-inference';
+
+    // Construct payloads
+    const builder = new PayloadBuilder();
+    const context = builder.build(config, deploymentTarget);
+
+    // Load and parse service models from registry
+    const parser = new ServiceModelParser();
+    const serviceModels = [];
+    try {
+        const entries = readdirSync(registryPath, { withFileTypes: true });
+        for (const entry of entries) {
+            if (entry.isDirectory()) {
+                const modelPath = path.join(registryPath, entry.name, 'service-2.json');
+                if (existsSync(modelPath)) {
+                    const rawModel = JSON.parse(readFileSync(modelPath, 'utf8'));
+                    serviceModels.push(parser.parse(rawModel));
+                }
+            }
+        }
+    } catch {
+        console.log('⚠️  Could not load service models from registry');
+        process.exit(2);
+        return 2;
+    }
+
+    // Run validation engine
+    const engine = new SchemaValidationEngine({
+        registryPath,
+        smartMode: smart,
+        serviceModels
+    });
+
+    const report = await engine.validate(context);
+    const summary = report.getSummary();
+
+    // Load manifest for version info
+    const manifest = loadManifest(registryPath);
+
+    // Output report
+    if (format === 'json') {
+        report.metadata.serviceModelVersionDate = manifest?.lastSynced || null;
+        const output = report.toJSON();
+        console.log(JSON.stringify(output, null, 2));
+    } else {
+        // Print static results immediately
+        const text = report.toText();
+        console.log(text);
+
+        // On success, print version info
+        if (summary.errors === 0) {
+            const versionDate = manifest?.lastSynced
+                ? new Date(manifest.lastSynced).toISOString().split('T')[0]
+                : 'unknown';
+            console.log('');
+            console.log('✅ Validation passed');
+            console.log(`   Service model version: ${versionDate}`);
+            console.log(`   Fields validated: ${summary.fieldsValidated}`);
+        }
+
+        // If smart mode and results are streaming, display them after static
+        if (smart && summary.advisory > 0) {
+            console.log('');
+            console.log('── Smart-mode findings ──');
+            for (const finding of report.advisoryFindings) {
+                console.log(`  ℹ ${finding.fieldPath || finding.operation}: ${finding.remediationHint || ''}`);
+            }
+        }
+    }
+
+    // Exit code
+    const exitCode = summary.errors > 0 ? 1 : 0;
+    process.exit(exitCode);
+    return exitCode;
+}
+
+export default { run, parseDoConfig };