@aifabrix/builder 2.6.3 → 2.7.0

package/lib/utils/schema-resolver.js

@@ -0,0 +1,174 @@
+/**
+ * Schema Resolution Utilities
+ *
+ * Resolves paths for external integration schemas from application configuration.
+ * Handles schemaBasePath resolution and external file discovery.
+ *
+ * @fileoverview Schema path resolution utilities for AI Fabrix Builder
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+/**
+ * Resolves schemaBasePath from application variables.yaml
+ * Supports both absolute and relative paths
+ *
+ * @async
+ * @function resolveSchemaBasePath
+ * @param {string} appName - Application name
+ * @returns {Promise<string>} Resolved absolute path to schema base directory
+ * @throws {Error} If variables.yaml not found, externalIntegration missing, or path invalid
+ *
+ * @example
+ * const basePath = await resolveSchemaBasePath('myapp');
+ * // Returns: '/path/to/builder/myapp/schemas'
+ */
+async function resolveSchemaBasePath(appName) {
+  if (!appName || typeof appName !== 'string') {
+    throw new Error('App name is required and must be a string');
+  }
+
+  const variablesPath = path.join(process.cwd(), 'builder', appName, 'variables.yaml');
+
+  if (!fs.existsSync(variablesPath)) {
+    throw new Error(`variables.yaml not found: ${variablesPath}`);
+  }
+
+  const content = fs.readFileSync(variablesPath, 'utf8');
+  let variables;
+
+  try {
+    variables = yaml.load(content);
+  } catch (error) {
+    throw new Error(`Invalid YAML syntax in variables.yaml: ${error.message}`);
+  }
+
+  // Check if externalIntegration block exists
+  if (!variables.externalIntegration) {
+    throw new Error(`externalIntegration block not found in variables.yaml for app: ${appName}`);
+  }
+
+  if (!variables.externalIntegration.schemaBasePath) {
+    throw new Error(`schemaBasePath not found in externalIntegration block for app: ${appName}`);
+  }
+
+  const schemaBasePath = variables.externalIntegration.schemaBasePath;
+  const variablesDir = path.dirname(variablesPath);
+
+  // Resolve path (absolute or relative to variables.yaml location)
+  let resolvedPath;
+  if (path.isAbsolute(schemaBasePath)) {
+    resolvedPath = schemaBasePath;
+  } else {
+    resolvedPath = path.resolve(variablesDir, schemaBasePath);
+  }
+
+  // Normalize path
+  resolvedPath = path.normalize(resolvedPath);
+
+  // Validate path exists
+  if (!fs.existsSync(resolvedPath)) {
+    throw new Error(`Schema base path does not exist: ${resolvedPath}`);
+  }
+
+  if (!fs.statSync(resolvedPath).isDirectory()) {
+    throw new Error(`Schema base path is not a directory: ${resolvedPath}`);
+  }
+
+  return resolvedPath;
+}
+
+/**
+ * Resolves all external system and datasource files from application configuration
+ * Returns array of file paths with metadata
+ *
+ * @async
+ * @function resolveExternalFiles
+ * @param {string} appName - Application name
+ * @returns {Promise<Array<{path: string, type: 'system'|'datasource', fileName: string}>>} Array of resolved file paths with metadata
+ * @throws {Error} If files cannot be resolved or do not exist
+ *
+ * @example
+ * const files = await resolveExternalFiles('myapp');
+ * // Returns: [
+ * //   { path: '/path/to/hubspot.json', type: 'system', fileName: 'hubspot.json' },
+ * //   { path: '/path/to/hubspot-deal.json', type: 'datasource', fileName: 'hubspot-deal.json' }
+ * // ]
+ */
+async function resolveExternalFiles(appName) {
+  if (!appName || typeof appName !== 'string') {
+    throw new Error('App name is required and must be a string');
+  }
+
+  const variablesPath = path.join(process.cwd(), 'builder', appName, 'variables.yaml');
+
+  if (!fs.existsSync(variablesPath)) {
+    throw new Error(`variables.yaml not found: ${variablesPath}`);
+  }
+
+  const content = fs.readFileSync(variablesPath, 'utf8');
+  let variables;
+
+  try {
+    variables = yaml.load(content);
+  } catch (error) {
+    throw new Error(`Invalid YAML syntax in variables.yaml: ${error.message}`);
+  }
+
+  // Check if externalIntegration block exists
+  if (!variables.externalIntegration) {
+    return []; // No external integration, return empty array
+  }
+
+  // Resolve schema base path
+  const schemaBasePath = await resolveSchemaBasePath(appName);
+  const resolvedFiles = [];
+
+  // Resolve systems files
+  if (variables.externalIntegration.systems && Array.isArray(variables.externalIntegration.systems)) {
+    for (const systemFile of variables.externalIntegration.systems) {
+      const systemPath = path.join(schemaBasePath, systemFile);
+      const normalizedPath = path.normalize(systemPath);
+
+      if (!fs.existsSync(normalizedPath)) {
+        throw new Error(`External system file not found: ${normalizedPath}`);
+      }
+
+      resolvedFiles.push({
+        path: normalizedPath,
+        type: 'system',
+        fileName: systemFile
+      });
+    }
+  }
+
+  // Resolve datasources files
+  if (variables.externalIntegration.dataSources && Array.isArray(variables.externalIntegration.dataSources)) {
+    for (const datasourceFile of variables.externalIntegration.dataSources) {
+      const datasourcePath = path.join(schemaBasePath, datasourceFile);
+      const normalizedPath = path.normalize(datasourcePath);
+
+      if (!fs.existsSync(normalizedPath)) {
+        throw new Error(`External datasource file not found: ${normalizedPath}`);
+      }
+
+      resolvedFiles.push({
+        path: normalizedPath,
+        type: 'datasource',
+        fileName: datasourceFile
+      });
+    }
+  }
+
+  return resolvedFiles;
+}
+
+module.exports = {
+  resolveSchemaBasePath,
+  resolveExternalFiles
+};
+

package/lib/utils/secrets-helpers.js

@@ -13,10 +13,10 @@ const path = require('path');
 const yaml = require('js-yaml');
 const config = require('../config');
 const { buildHostnameToServiceMap, resolveUrlPort } = require('./secrets-utils');
-const { rewriteInfraEndpoints, getEnvHosts } = require('./env-endpoints');
+const { rewriteInfraEndpoints, getEnvHosts, getServicePort, getServiceHost, getLocalhostOverride } = require('./env-endpoints');
 const { loadEnvConfig } = require('./env-config-loader');
-const { processEnvVariables } = require('./env-copy');
 const { updateContainerPortInEnvFile } = require('./env-ports');
+const { buildEnvVarMap } = require('./env-map');
 
 /**
  * Interpolate ${VAR} occurrences with values from envVars map
@@ -262,6 +262,24 @@ async function adjustLocalEnvPortsInContent(envContent, variablesPath) {
   // Update infra endpoints with developer-id adjusted ports for local context
   updated = await rewriteInfraEndpoints(updated, 'local');
 
+  // Interpolate ${VAR} references created by rewriteInfraEndpoints
+  // Get the ports that were just set by rewriteInfraEndpoints for interpolation
+  const hostsForPorts = await getEnvHosts('local');
+  const redisPort = await getServicePort('REDIS_PORT', 'redis', hostsForPorts, 'local');
+  const dbPort = await getServicePort('DB_PORT', 'postgres', hostsForPorts, 'local');
+  const localhostOverride = getLocalhostOverride('local');
+  const redisHost = getServiceHost(hostsForPorts.REDIS_HOST, 'local', 'localhost', localhostOverride);
+  const dbHost = getServiceHost(hostsForPorts.DB_HOST, 'local', 'localhost', localhostOverride);
+
+  // Build envVars map and ensure it has the correct values
+  const envVars = await buildEnvVarMap('local', null, devIdNum);
+  // Override with the actual values that were just set by rewriteInfraEndpoints
+  envVars.REDIS_HOST = redisHost;
+  envVars.REDIS_PORT = String(redisPort);
+  envVars.DB_HOST = dbHost;
+  envVars.DB_PORT = String(dbPort);
+  updated = interpolateEnvVars(updated, envVars);
+
   return updated;
 }
 
@@ -276,6 +294,29 @@ function readYamlAtPath(filePath) {
   return yaml.load(content);
 }
 
+/**
+ * Merge a single secret value from canonical into result
+ * @function mergeSecretValue
+ * @param {Object} result - Result object to merge into
+ * @param {string} key - Secret key
+ * @param {*} canonicalValue - Value from canonical secrets
+ */
+function mergeSecretValue(result, key, canonicalValue) {
+  const currentValue = result[key];
+  // Fill missing, empty, or undefined values
+  if (!(key in result) || currentValue === undefined || currentValue === null || currentValue === '') {
+    result[key] = canonicalValue;
+    return;
+  }
+  // Only replace values that are encrypted (have secure:// prefix)
+  // Plaintext values (no secure://) are used as-is
+  if (typeof currentValue === 'string' && typeof canonicalValue === 'string') {
+    if (currentValue.startsWith('secure://')) {
+      result[key] = canonicalValue;
+    }
+  }
+}
+
 /**
  * Apply canonical secrets path override if configured and file exists
  * @async
@@ -287,21 +328,29 @@ async function applyCanonicalSecretsOverride(currentSecrets) {
   let mergedSecrets = currentSecrets || {};
   try {
     const canonicalPath = await config.getSecretsPath();
-    if (canonicalPath) {
-      const resolvedCanonical = path.isAbsolute(canonicalPath)
-        ? canonicalPath
-        : path.resolve(process.cwd(), canonicalPath);
-      if (fs.existsSync(resolvedCanonical)) {
-        const configSecrets = readYamlAtPath(resolvedCanonical);
-        // Apply canonical secrets as a fallback source:
-        // - Do NOT override any existing keys from user/build
-        // - Add only missing keys from canonical path
-        if (configSecrets && typeof configSecrets === 'object') {
-          const result = { ...configSecrets, ...mergedSecrets };
-          mergedSecrets = result;
-        }
-      }
+    if (!canonicalPath) {
+      return mergedSecrets;
+    }
+    const resolvedCanonical = path.isAbsolute(canonicalPath)
+      ? canonicalPath
+      : path.resolve(process.cwd(), canonicalPath);
+    if (!fs.existsSync(resolvedCanonical)) {
+      return mergedSecrets;
+    }
+    const configSecrets = readYamlAtPath(resolvedCanonical);
+    if (!configSecrets || typeof configSecrets !== 'object') {
+      return mergedSecrets;
+    }
+    // Apply canonical secrets as a fallback source:
+    // - Do NOT override any existing keys from user/build
+    // - Add only missing keys from canonical path
+    // - Also fill in empty/undefined values from canonical path
+    // - Replace encrypted values (secure://) with canonical plaintext
+    const result = { ...mergedSecrets };
+    for (const [key, canonicalValue] of Object.entries(configSecrets)) {
+      mergeSecretValue(result, key, canonicalValue);
     }
+    mergedSecrets = result;
   } catch {
     // ignore and fall through
   }
@@ -351,7 +400,6 @@ module.exports = {
   replaceKvInContent,
   resolveServicePortsInEnvContent,
   loadEnvTemplate,
-  processEnvVariables,
   updateContainerPortInEnvFile,
   adjustLocalEnvPortsInContent,
   readYamlAtPath,

package/lib/validate.js

@@ -0,0 +1,299 @@
+/**
+ * Main Validation Command
+ *
+ * Validates applications or external integration files.
+ * Supports app name validation (including externalIntegration block) or direct file validation.
+ *
+ * @fileoverview Main validation command for AI Fabrix Builder
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fs = require('fs');
+const path = require('path');
+const chalk = require('chalk');
+const validator = require('./validator');
+const { resolveExternalFiles } = require('./utils/schema-resolver');
+const { loadExternalSystemSchema, loadExternalDataSourceSchema, detectSchemaType } = require('./utils/schema-loader');
+const { formatValidationErrors } = require('./utils/error-formatter');
+const logger = require('./utils/logger');
+
+/**
+ * Validates a single external file against its schema
+ *
+ * @async
+ * @function validateExternalFile
+ * @param {string} filePath - Path to the file
+ * @param {string} type - File type: 'system' | 'datasource'
+ * @returns {Promise<Object>} Validation result
+ */
+async function validateExternalFile(filePath, type) {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`File not found: ${filePath}`);
+  }
+
+  const content = fs.readFileSync(filePath, 'utf8');
+  let parsed;
+
+  try {
+    parsed = JSON.parse(content);
+  } catch (error) {
+    return {
+      valid: false,
+      errors: [`Invalid JSON syntax: ${error.message}`],
+      warnings: []
+    };
+  }
+
+  let validate;
+  if (type === 'system') {
+    validate = loadExternalSystemSchema();
+  } else if (type === 'datasource') {
+    validate = loadExternalDataSourceSchema();
+  } else {
+    throw new Error(`Unknown file type: ${type}`);
+  }
+
+  const valid = validate(parsed);
+
+  return {
+    valid,
+    errors: valid ? [] : formatValidationErrors(validate.errors),
+    warnings: []
+  };
+}
+
+/**
+ * Validates application or external integration file
+ * Detects if input is app name or file path and validates accordingly
+ *
+ * @async
+ * @function validateAppOrFile
+ * @param {string} appOrFile - Application name or file path
+ * @returns {Promise<Object>} Validation result with aggregated results
+ * @throws {Error} If validation fails
+ *
+ * @example
+ * const result = await validateAppOrFile('myapp');
+ * // Returns: { valid: true, application: {...}, externalFiles: [...] }
+ */
+async function validateAppOrFile(appOrFile) {
+  if (!appOrFile || typeof appOrFile !== 'string') {
+    throw new Error('App name or file path is required');
+  }
+
+  // Check if it's a file path (exists and is a file)
+  const isFilePath = fs.existsSync(appOrFile) && fs.statSync(appOrFile).isFile();
+
+  if (isFilePath) {
+    // Validate single file
+    const schemaType = detectSchemaType(appOrFile);
+    let result;
+
+    if (schemaType === 'application') {
+      // For application files, we'd need to transform them first
+      // For now, just validate JSON structure
+      const content = fs.readFileSync(appOrFile, 'utf8');
+      try {
+        JSON.parse(content);
+      } catch (error) {
+        return {
+          valid: false,
+          errors: [`Invalid JSON syntax: ${error.message}`],
+          warnings: []
+        };
+      }
+      // Note: Full application validation requires variables.yaml transformation
+      // This is a simplified validation for direct JSON files
+      result = {
+        valid: true,
+        errors: [],
+        warnings: ['Application file validation is simplified. Use app name for full validation.']
+      };
+    } else if (schemaType === 'external-system') {
+      result = await validateExternalFile(appOrFile, 'system');
+    } else if (schemaType === 'external-datasource') {
+      result = await validateExternalFile(appOrFile, 'datasource');
+    } else {
+      throw new Error(`Unknown schema type: ${schemaType}`);
+    }
+
+    return {
+      valid: result.valid,
+      file: appOrFile,
+      type: schemaType,
+      errors: result.errors,
+      warnings: result.warnings
+    };
+  }
+
+  // Treat as app name
+  const appName = appOrFile;
+
+  // Validate application
+  const appValidation = await validator.validateApplication(appName);
+
+  // Check for externalIntegration block
+  const variablesPath = path.join(process.cwd(), 'builder', appName, 'variables.yaml');
+  if (!fs.existsSync(variablesPath)) {
+    return {
+      valid: appValidation.valid,
+      application: appValidation,
+      externalFiles: []
+    };
+  }
+
+  const yamlLib = require('js-yaml');
+  const content = fs.readFileSync(variablesPath, 'utf8');
+  let variables;
+
+  try {
+    variables = yamlLib.load(content);
+  } catch (error) {
+    return {
+      valid: appValidation.valid,
+      application: appValidation,
+      externalFiles: [],
+      warnings: [`Could not parse variables.yaml to check externalIntegration: ${error.message}`]
+    };
+  }
+
+  // If no externalIntegration block, return app validation only
+  if (!variables.externalIntegration) {
+    return {
+      valid: appValidation.valid,
+      application: appValidation,
+      externalFiles: []
+    };
+  }
+
+  // Resolve and validate external files
+  const externalFiles = await resolveExternalFiles(appName);
+  const externalValidations = [];
+
+  for (const fileInfo of externalFiles) {
+    try {
+      const validation = await validateExternalFile(fileInfo.path, fileInfo.type);
+      externalValidations.push({
+        file: fileInfo.fileName,
+        path: fileInfo.path,
+        type: fileInfo.type,
+        ...validation
+      });
+    } catch (error) {
+      externalValidations.push({
+        file: fileInfo.fileName,
+        path: fileInfo.path,
+        type: fileInfo.type,
+        valid: false,
+        errors: [error.message],
+        warnings: []
+      });
+    }
+  }
+
+  // Aggregate results
+  const allValid = appValidation.valid && externalValidations.every(v => v.valid);
+  const allErrors = [
+    ...appValidation.errors,
+    ...externalValidations.flatMap(v => v.errors.map(e => `${v.file}: ${e}`))
+  ];
+  const allWarnings = [
+    ...appValidation.warnings,
+    ...externalValidations.flatMap(v => v.warnings)
+  ];
+
+  return {
+    valid: allValid,
+    application: appValidation,
+    externalFiles: externalValidations,
+    errors: allErrors,
+    warnings: allWarnings
+  };
+}
+
+/**
+ * Displays validation results in a user-friendly format
+ *
+ * @function displayValidationResults
+ * @param {Object} result - Validation result from validateAppOrFile
+ */
+function displayValidationResults(result) {
+  if (result.valid) {
+    logger.log(chalk.green('\n✓ Validation passed!'));
+  } else {
+    logger.log(chalk.red('\n✗ Validation failed!'));
+  }
+
+  // Display application validation
+  if (result.application) {
+    logger.log(chalk.blue('\nApplication:'));
+    if (result.application.valid) {
+      logger.log(chalk.green('  ✓ Application configuration is valid'));
+    } else {
+      logger.log(chalk.red('  ✗ Application configuration has errors:'));
+      result.application.errors.forEach(error => {
+        logger.log(chalk.red(`    • ${error}`));
+      });
+    }
+    if (result.application.warnings && result.application.warnings.length > 0) {
+      result.application.warnings.forEach(warning => {
+        logger.log(chalk.yellow(`    ⚠ ${warning}`));
+      });
+    }
+  }
+
+  // Display external files validation
+  if (result.externalFiles && result.externalFiles.length > 0) {
+    logger.log(chalk.blue('\nExternal Integration Files:'));
+    result.externalFiles.forEach(file => {
+      if (file.valid) {
+        logger.log(chalk.green(`  ✓ ${file.file} (${file.type})`));
+      } else {
+        logger.log(chalk.red(`  ✗ ${file.file} (${file.type}):`));
+        file.errors.forEach(error => {
+          logger.log(chalk.red(`    • ${error}`));
+        });
+      }
+      if (file.warnings && file.warnings.length > 0) {
+        file.warnings.forEach(warning => {
+          logger.log(chalk.yellow(`    ⚠ ${warning}`));
+        });
+      }
+    });
+  }
+
+  // Display file validation (for direct file validation)
+  if (result.file) {
+    logger.log(chalk.blue(`\nFile: ${result.file}`));
+    logger.log(chalk.blue(`Type: ${result.type}`));
+    if (result.valid) {
+      logger.log(chalk.green('  ✓ File is valid'));
+    } else {
+      logger.log(chalk.red('  ✗ File has errors:'));
+      result.errors.forEach(error => {
+        logger.log(chalk.red(`    • ${error}`));
+      });
+    }
+    if (result.warnings && result.warnings.length > 0) {
+      result.warnings.forEach(warning => {
+        logger.log(chalk.yellow(`    ⚠ ${warning}`));
+      });
+    }
+  }
+
+  // Display aggregated warnings
+  if (result.warnings && result.warnings.length > 0) {
+    logger.log(chalk.yellow('\nWarnings:'));
+    result.warnings.forEach(warning => {
+      logger.log(chalk.yellow(`  • ${warning}`));
+    });
+  }
+}
+
+module.exports = {
+  validateAppOrFile,
+  displayValidationResults,
+  validateExternalFile
+};
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aifabrix/builder",
-  "version": "2.6.3",
+  "version": "2.7.0",
   "description": "AI Fabrix Local Fabric & Deployment SDK",
   "main": "lib/index.js",
   "bin": {