@aifabrix/builder 2.6.3 → 2.7.0

package/bin/aifabrix.js

@@ -16,6 +16,7 @@
 const { Command } = require('commander');
 const cli = require('../lib/cli');
 const { setupAppCommands } = require('../lib/commands/app');
+const { setupDatasourceCommands } = require('../lib/commands/datasource');
 const logger = require('../lib/utils/logger');
 const packageJson = require('../package.json');
 
@@ -36,6 +37,9 @@ function initializeCLI() {
   // Add application management commands
   setupAppCommands(program);
 
+  // Add datasource management commands
+  setupDatasourceCommands(program);
+
   // Parse command line arguments
   program.parse();
 }

package/lib/cli.js

@@ -1,3 +1,4 @@
+/* eslint-disable max-lines */
 /**
  * AI Fabrix Builder CLI Command Definitions
  *
@@ -265,12 +266,26 @@ function setupCommands(program) {
 
   // Utility commands
   program.command('resolve <app>')
-    .description('Generate .env file from template')
+    .description('Generate .env file from template and validate application files')
     .option('-f, --force', 'Generate missing secret keys in secrets file')
+    .option('--skip-validation', 'Skip file validation after generating .env')
     .action(async(appName, options) => {
       try {
-        const envPath = await secrets.generateEnvFile(appName, undefined, 'local', options.force);
+        // builder/.env should use docker context (postgres:5432)
+        // apps/.env (if envOutputPath is set) will be generated with local context by processEnvVariables
+        const envPath = await secrets.generateEnvFile(appName, undefined, 'docker', options.force);
         logger.log(`✓ Generated .env file: ${envPath}`);
+
+        // Validate application files after generating .env
+        if (!options.skipValidation) {
+          const validate = require('./validate');
+          const result = await validate.validateAppOrFile(appName);
+          validate.displayValidationResults(result);
+          if (!result.valid) {
+            logger.log(chalk.yellow('\n⚠️  Validation found errors. Fix them before deploying.'));
+            process.exit(1);
+          }
+        }
       } catch (error) {
         handleCommandError(error, 'resolve');
         process.exit(1);
@@ -330,6 +345,40 @@ function setupCommands(program) {
       }
     });
 
+  // Validation command
+  program.command('validate <appOrFile>')
+    .description('Validate application or external integration file')
+    .action(async(appOrFile) => {
+      try {
+        const validate = require('./validate');
+        const result = await validate.validateAppOrFile(appOrFile);
+        validate.displayValidationResults(result);
+        if (!result.valid) {
+          process.exit(1);
+        }
+      } catch (error) {
+        handleCommandError(error, 'validate');
+        process.exit(1);
+      }
+    });
+
+  // Diff command
+  program.command('diff <file1> <file2>')
+    .description('Compare two configuration files (for deployment pipeline)')
+    .action(async(file1, file2) => {
+      try {
+        const diff = require('./diff');
+        const result = await diff.compareFiles(file1, file2);
+        diff.formatDiffOutput(result);
+        if (!result.identical) {
+          process.exit(1);
+        }
+      } catch (error) {
+        handleCommandError(error, 'diff');
+        process.exit(1);
+      }
+    });
+
   program.command('dockerfile <app>')
     .description('Generate Dockerfile for an application')
     .option('-l, --language <lang>', 'Override language detection')

package/lib/commands/datasource.js

@@ -0,0 +1,94 @@
+/**
+ * AI Fabrix Builder - Datasource Commands
+ *
+ * Handles datasource validation, listing, comparison, and deployment
+ * Commands: datasource validate, datasource list, datasource diff, datasource deploy
+ *
+ * @fileoverview Datasource management commands for AI Fabrix Builder
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const { validateDatasourceFile } = require('../datasource-validate');
+const { listDatasources } = require('../datasource-list');
+const { compareDatasources } = require('../datasource-diff');
+const { deployDatasource } = require('../datasource-deploy');
+
+/**
+ * Setup datasource management commands
+ * @param {Command} program - Commander program instance
+ */
+function setupDatasourceCommands(program) {
+  const datasource = program
+    .command('datasource')
+    .description('Manage external data sources');
+
+  // Validate command
+  datasource
+    .command('validate <file>')
+    .description('Validate external datasource JSON file')
+    .action(async(file) => {
+      try {
+        const result = await validateDatasourceFile(file);
+        if (result.valid) {
+          logger.log(chalk.green(`\n✓ Datasource file is valid: ${file}`));
+        } else {
+          logger.log(chalk.red(`\n✗ Datasource file has errors: ${file}`));
+          result.errors.forEach(error => {
+            logger.log(chalk.red(`  • ${error}`));
+          });
+          process.exit(1);
+        }
+      } catch (error) {
+        logger.error(chalk.red('❌ Validation failed:'), error.message);
+        process.exit(1);
+      }
+    });
+
+  // List command
+  datasource
+    .command('list')
+    .description('List datasources from environment')
+    .requiredOption('-e, --environment <env>', 'Environment ID or key')
+    .action(async(options) => {
+      try {
+        await listDatasources(options);
+      } catch (error) {
+        logger.error(chalk.red('❌ Failed to list datasources:'), error.message);
+        process.exit(1);
+      }
+    });
+
+  // Diff command
+  datasource
+    .command('diff <file1> <file2>')
+    .description('Compare two datasource configuration files (for dataplane)')
+    .action(async(file1, file2) => {
+      try {
+        await compareDatasources(file1, file2);
+      } catch (error) {
+        logger.error(chalk.red('❌ Diff failed:'), error.message);
+        process.exit(1);
+      }
+    });
+
+  // Deploy command
+  datasource
+    .command('deploy <myapp> <file>')
+    .description('Deploy datasource to dataplane')
+    .requiredOption('--controller <url>', 'Controller URL')
+    .requiredOption('-e, --environment <env>', 'Environment (miso, dev, tst, pro)')
+    .action(async(myapp, file, options) => {
+      try {
+        await deployDatasource(myapp, file, options);
+      } catch (error) {
+        logger.error(chalk.red('❌ Deployment failed:'), error.message);
+        process.exit(1);
+      }
+    });
+}
+
+module.exports = { setupDatasourceCommands };
+

package/lib/datasource-deploy.js

@@ -0,0 +1,182 @@
+/**
+ * Datasource Deployment
+ *
+ * Deploys datasource to dataplane via controller API.
+ * Gets dataplane URL from controller, then deploys to dataplane.
+ *
+ * @fileoverview Datasource deployment for AI Fabrix Builder
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fs = require('fs');
+const chalk = require('chalk');
+const { getDeploymentAuth } = require('./utils/token-manager');
+const { authenticatedApiCall } = require('./utils/api');
+const { formatApiError } = require('./utils/api-error-handler');
+const logger = require('./utils/logger');
+const { validateDatasourceFile } = require('./datasource-validate');
+
+/**
+ * Gets dataplane URL from controller by fetching application details
+ *
+ * @async
+ * @function getDataplaneUrl
+ * @param {string} controllerUrl - Controller URL
+ * @param {string} appKey - Application key
+ * @param {string} environment - Environment key
+ * @param {Object} authConfig - Authentication configuration
+ * @returns {Promise<string>} Dataplane URL
+ * @throws {Error} If dataplane URL cannot be retrieved
+ */
+async function getDataplaneUrl(controllerUrl, appKey, environment, authConfig) {
+  // Call controller API to get application details
+  // Expected: GET /api/v1/environments/{env}/applications/{appKey}
+  const endpoint = `${controllerUrl}/api/v1/environments/${environment}/applications/${appKey}`;
+
+  let response;
+  if (authConfig.type === 'bearer' && authConfig.token) {
+    response = await authenticatedApiCall(endpoint, {}, authConfig.token);
+  } else {
+    // For credentials, we'd need to use a different API call method
+    // For now, use bearer token approach
+    throw new Error('Bearer token authentication required for getting dataplane URL');
+  }
+
+  if (!response.success || !response.data) {
+    const formattedError = response.formattedError || formatApiError(response);
+    throw new Error(`Failed to get application from controller: ${formattedError}`);
+  }
+
+  // Extract dataplane URL from application response
+  // This is a placeholder - actual response structure may vary
+  const application = response.data.data || response.data;
+  const dataplaneUrl = application.dataplaneUrl || application.dataplane?.url || application.configuration?.dataplaneUrl;
+
+  if (!dataplaneUrl) {
+    logger.error(chalk.red('❌ Dataplane URL not found in application response'));
+    logger.error(chalk.gray('\nApplication response:'));
+    logger.error(chalk.gray(JSON.stringify(application, null, 2)));
+    throw new Error('Dataplane URL not found in application configuration');
+  }
+
+  return dataplaneUrl;
+}
+
+/**
+ * Deploys datasource to dataplane
+ *
+ * @async
+ * @function deployDatasource
+ * @param {string} appKey - Application key
+ * @param {string} filePath - Path to datasource JSON file
+ * @param {Object} options - Deployment options
+ * @param {string} options.controller - Controller URL
+ * @param {string} options.environment - Environment key
+ * @returns {Promise<Object>} Deployment result
+ * @throws {Error} If deployment fails
+ */
+async function deployDatasource(appKey, filePath, options) {
+  if (!appKey || typeof appKey !== 'string') {
+    throw new Error('Application key is required');
+  }
+  if (!filePath || typeof filePath !== 'string') {
+    throw new Error('File path is required');
+  }
+  if (!options.controller) {
+    throw new Error('Controller URL is required (--controller)');
+  }
+  if (!options.environment) {
+    throw new Error('Environment is required (-e, --environment)');
+  }
+
+  logger.log(chalk.blue('📋 Deploying datasource...\n'));
+
+  // Validate datasource file
+  logger.log(chalk.blue('🔍 Validating datasource file...'));
+  const validation = await validateDatasourceFile(filePath);
+  if (!validation.valid) {
+    logger.error(chalk.red('❌ Datasource validation failed:'));
+    validation.errors.forEach(error => {
+      logger.error(chalk.red(`  • ${error}`));
+    });
+    throw new Error('Datasource file validation failed');
+  }
+  logger.log(chalk.green('✓ Datasource file is valid'));
+
+  // Load datasource configuration
+  const content = fs.readFileSync(filePath, 'utf8');
+  let datasourceConfig;
+  try {
+    datasourceConfig = JSON.parse(content);
+  } catch (error) {
+    throw new Error(`Failed to parse datasource file: ${error.message}`);
+  }
+
+  // Extract systemKey
+  const systemKey = datasourceConfig.systemKey;
+  if (!systemKey) {
+    throw new Error('systemKey is required in datasource configuration');
+  }
+
+  // Get authentication
+  logger.log(chalk.blue('🔐 Getting authentication...'));
+  const authConfig = await getDeploymentAuth(options.controller, options.environment, appKey);
+  logger.log(chalk.green('✓ Authentication successful'));
+
+  // Get dataplane URL from controller
+  logger.log(chalk.blue('🌐 Getting dataplane URL from controller...'));
+  const dataplaneUrl = await getDataplaneUrl(options.controller, appKey, options.environment, authConfig);
+  logger.log(chalk.green(`✓ Dataplane URL: ${dataplaneUrl}`));
+
+  // Deploy to dataplane
+  logger.log(chalk.blue('\n🚀 Deploying to dataplane...'));
+  const deployEndpoint = `${dataplaneUrl}/api/v1/pipeline/${systemKey}/deploy`;
+
+  // Prepare deployment request
+  const deployRequest = {
+    datasource: datasourceConfig
+  };
+
+  // Make API call to dataplane
+  // This is a placeholder - actual API structure may vary
+  let deployResponse;
+  if (authConfig.type === 'bearer' && authConfig.token) {
+    deployResponse = await authenticatedApiCall(
+      deployEndpoint,
+      {
+        method: 'POST',
+        body: JSON.stringify(deployRequest)
+      },
+      authConfig.token
+    );
+  } else {
+    throw new Error('Bearer token authentication required for dataplane deployment');
+  }
+
+  if (!deployResponse.success) {
+    const formattedError = deployResponse.formattedError || formatApiError(deployResponse);
+    logger.error(chalk.red('❌ Deployment failed:'));
+    logger.error(formattedError);
+    throw new Error(`Dataplane deployment failed: ${formattedError}`);
+  }
+
+  logger.log(chalk.green('\n✓ Datasource deployed successfully!'));
+  logger.log(chalk.blue(`\nDatasource: ${datasourceConfig.key || datasourceConfig.displayName}`));
+  logger.log(chalk.blue(`System: ${systemKey}`));
+  logger.log(chalk.blue(`Environment: ${options.environment}`));
+
+  return {
+    success: true,
+    datasourceKey: datasourceConfig.key,
+    systemKey: systemKey,
+    environment: options.environment,
+    dataplaneUrl: dataplaneUrl
+  };
+}
+
+module.exports = {
+  deployDatasource,
+  getDataplaneUrl
+};
+

package/lib/datasource-diff.js

@@ -0,0 +1,73 @@
+/**
+ * Datasource Diff Command
+ *
+ * Compares two datasource configuration files.
+ * Specialized for dataplane deployment validation.
+ *
+ * @fileoverview Datasource comparison for AI Fabrix Builder
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const chalk = require('chalk');
+const { compareFiles, formatDiffOutput } = require('./diff');
+const logger = require('./utils/logger');
+
+/**
+ * Compares two datasource files with focus on dataplane-relevant fields
+ *
+ * @async
+ * @function compareDatasources
+ * @param {string} file1 - Path to first datasource file
+ * @param {string} file2 - Path to second datasource file
+ * @returns {Promise<void>}
+ * @throws {Error} If comparison fails
+ */
+async function compareDatasources(file1, file2) {
+  const result = await compareFiles(file1, file2);
+
+  // Filter and highlight dataplane-relevant changes
+  const dataplaneRelevant = {
+    fieldMappings: result.changed.filter(c => c.path.includes('fieldMappings')),
+    exposed: result.changed.filter(c => c.path.includes('exposed')),
+    sync: result.changed.filter(c => c.path.includes('sync')),
+    openapi: result.changed.filter(c => c.path.includes('openapi')),
+    mcp: result.changed.filter(c => c.path.includes('mcp'))
+  };
+
+  // Display standard diff
+  formatDiffOutput(result);
+
+  // Display dataplane-specific highlights
+  const hasDataplaneChanges = Object.values(dataplaneRelevant).some(arr => arr.length > 0);
+
+  if (hasDataplaneChanges) {
+    logger.log(chalk.blue('\n📊 Dataplane-Relevant Changes:'));
+
+    if (dataplaneRelevant.fieldMappings.length > 0) {
+      logger.log(chalk.yellow(`  • Field Mappings: ${dataplaneRelevant.fieldMappings.length} changes`));
+    }
+    if (dataplaneRelevant.exposed.length > 0) {
+      logger.log(chalk.yellow(`  • Exposed Fields: ${dataplaneRelevant.exposed.length} changes`));
+    }
+    if (dataplaneRelevant.sync.length > 0) {
+      logger.log(chalk.yellow(`  • Sync Configuration: ${dataplaneRelevant.sync.length} changes`));
+    }
+    if (dataplaneRelevant.openapi.length > 0) {
+      logger.log(chalk.yellow(`  • OpenAPI Configuration: ${dataplaneRelevant.openapi.length} changes`));
+    }
+    if (dataplaneRelevant.mcp.length > 0) {
+      logger.log(chalk.yellow(`  • MCP Configuration: ${dataplaneRelevant.mcp.length} changes`));
+    }
+  }
+
+  // Exit with appropriate code
+  if (!result.identical) {
+    process.exit(1);
+  }
+}
+
+module.exports = {
+  compareDatasources
+};
+

package/lib/datasource-list.js

@@ -0,0 +1,138 @@
+/**
+ * Datasource List Command
+ *
+ * Lists datasources from an environment via controller API.
+ *
+ * @fileoverview Datasource listing for AI Fabrix Builder
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const chalk = require('chalk');
+const { getConfig } = require('./config');
+const { getOrRefreshDeviceToken } = require('./utils/token-manager');
+const { authenticatedApiCall } = require('./utils/api');
+const { formatApiError } = require('./utils/api-error-handler');
+const logger = require('./utils/logger');
+
+/**
+ * Extracts datasources array from API response
+ * Handles multiple response formats similar to applications list
+ *
+ * @function extractDatasources
+ * @param {Object} response - API response from authenticatedApiCall
+ * @returns {Array} Array of datasources
+ * @throws {Error} If response format is invalid
+ */
+function extractDatasources(response) {
+  const apiResponse = response.data;
+  let datasources;
+
+  // Check if apiResponse.data is an array (wrapped format)
+  if (apiResponse && apiResponse.data && Array.isArray(apiResponse.data)) {
+    datasources = apiResponse.data;
+  } else if (Array.isArray(apiResponse)) {
+    // Check if apiResponse is directly an array
+    datasources = apiResponse;
+  } else if (apiResponse && Array.isArray(apiResponse.items)) {
+    // Check if apiResponse.items is an array (paginated format)
+    datasources = apiResponse.items;
+  } else if (apiResponse && apiResponse.data && apiResponse.data.items && Array.isArray(apiResponse.data.items)) {
+    // Check if apiResponse.data.items is an array (wrapped paginated format)
+    datasources = apiResponse.data.items;
+  } else {
+    logger.error(chalk.red('❌ Invalid response: expected data array or items array'));
+    logger.error(chalk.gray('\nAPI response type:'), typeof apiResponse);
+    logger.error(chalk.gray('API response:'), JSON.stringify(apiResponse, null, 2));
+    throw new Error('Invalid API response format: expected array of datasources');
+  }
+
+  return datasources;
+}
+
+/**
+ * Displays datasources in a formatted table
+ *
+ * @function displayDatasources
+ * @param {Array} datasources - Array of datasource objects
+ * @param {string} environment - Environment key
+ */
+function displayDatasources(datasources, environment) {
+  if (datasources.length === 0) {
+    logger.log(chalk.yellow(`\nNo datasources found in environment: ${environment}`));
+    return;
+  }
+
+  logger.log(chalk.blue(`\n📋 Datasources in environment: ${environment}\n`));
+  logger.log(chalk.gray('Key'.padEnd(30) + 'Display Name'.padEnd(30) + 'System Key'.padEnd(20) + 'Version'.padEnd(15) + 'Status'));
+  logger.log(chalk.gray('-'.repeat(120)));
+
+  datasources.forEach((ds) => {
+    const key = (ds.key || 'N/A').padEnd(30);
+    const displayName = (ds.displayName || 'N/A').padEnd(30);
+    const systemKey = (ds.systemKey || 'N/A').padEnd(20);
+    const version = (ds.version || 'N/A').padEnd(15);
+    const status = ds.enabled !== false ? chalk.green('enabled') : chalk.red('disabled');
+    logger.log(`${key}${displayName}${systemKey}${version}${status}`);
+  });
+  logger.log('');
+}
+
+/**
+ * Lists datasources from an environment
+ *
+ * @async
+ * @function listDatasources
+ * @param {Object} options - Command options
+ * @param {string} options.environment - Environment ID or key
+ * @throws {Error} If listing fails
+ */
+async function listDatasources(options) {
+  const config = await getConfig();
+
+  // Try to get device token
+  let controllerUrl = null;
+  let token = null;
+
+  if (config.device) {
+    const deviceUrls = Object.keys(config.device);
+    if (deviceUrls.length > 0) {
+      controllerUrl = deviceUrls[0];
+      const deviceToken = await getOrRefreshDeviceToken(controllerUrl);
+      if (deviceToken && deviceToken.token) {
+        token = deviceToken.token;
+        controllerUrl = deviceToken.controller;
+      }
+    }
+  }
+
+  if (!token || !controllerUrl) {
+    logger.error(chalk.red('❌ Not logged in. Run: aifabrix login'));
+    logger.error(chalk.gray('   Use device code flow: aifabrix login --method device --controller <url>'));
+    process.exit(1);
+  }
+
+  // Call controller API - using placeholder endpoint until full specs available
+  // Expected: GET /api/v1/environments/{env}/datasources
+  const endpoint = `${controllerUrl}/api/v1/environments/${options.environment}/datasources`;
+  const response = await authenticatedApiCall(endpoint, {}, token);
+
+  if (!response.success || !response.data) {
+    const formattedError = response.formattedError || formatApiError(response);
+    logger.error(formattedError);
+    logger.error(chalk.gray('\nFull response for debugging:'));
+    logger.error(chalk.gray(JSON.stringify(response, null, 2)));
+    process.exit(1);
+    return; // Ensure we don't continue after exit
+  }
+
+  const datasources = extractDatasources(response);
+  displayDatasources(datasources, options.environment);
+}
+
+module.exports = {
+  listDatasources,
+  displayDatasources,
+  extractDatasources
+};
+

package/lib/datasource-validate.js

@@ -0,0 +1,63 @@
+/**
+ * Datasource Validation
+ *
+ * Validates external datasource JSON files against schema.
+ *
+ * @fileoverview Datasource validation for AI Fabrix Builder
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fs = require('fs');
+const { loadExternalDataSourceSchema } = require('./utils/schema-loader');
+const { formatValidationErrors } = require('./utils/error-formatter');
+
+/**
+ * Validates a datasource file against external-datasource schema
+ *
+ * @async
+ * @function validateDatasourceFile
+ * @param {string} filePath - Path to the datasource JSON file
+ * @returns {Promise<Object>} Validation result with errors and warnings
+ * @throws {Error} If file cannot be read or parsed
+ *
+ * @example
+ * const result = await validateDatasourceFile('./hubspot-deal.json');
+ * // Returns: { valid: true, errors: [], warnings: [] }
+ */
+async function validateDatasourceFile(filePath) {
+  if (!filePath || typeof filePath !== 'string') {
+    throw new Error('File path is required and must be a string');
+  }
+
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`File not found: ${filePath}`);
+  }
+
+  const content = fs.readFileSync(filePath, 'utf8');
+  let parsed;
+
+  try {
+    parsed = JSON.parse(content);
+  } catch (error) {
+    return {
+      valid: false,
+      errors: [`Invalid JSON syntax: ${error.message}`],
+      warnings: []
+    };
+  }
+
+  const validate = loadExternalDataSourceSchema();
+  const valid = validate(parsed);
+
+  return {
+    valid,
+    errors: valid ? [] : formatValidationErrors(validate.errors),
+    warnings: []
+  };
+}
+
+module.exports = {
+  validateDatasourceFile
+};
+