@aifabrix/builder 2.31.1 → 2.32.1

package/lib/external-system/deploy-helpers.js

@@ -0,0 +1,145 @@
+/**
+ * External System Deployment Helpers
+ *
+ * Helper functions for external system deployment validation
+ *
+ * @fileoverview Deployment helper utilities for external system deployment
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fs = require('fs').promises;
+const fsSync = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+const { detectAppType, getDeployJsonPath } = require('../utils/paths');
+
+/**
+ * Loads variables.yaml for an application
+ * @async
+ * @function loadVariablesYaml
+ * @param {string} appName - Application name
+ * @returns {Promise<Object>} Variables configuration
+ * @throws {Error} If file cannot be loaded
+ */
+async function loadVariablesYaml(appName) {
+  // Detect app type and get correct path (integration or builder)
+  const { appPath } = await detectAppType(appName);
+  const variablesPath = path.join(appPath, 'variables.yaml');
+  const content = await fs.readFile(variablesPath, 'utf8');
+  return yaml.load(content);
+}
+
+/**
+ * Validates a single system file
+ * @async
+ * @function validateSingleSystemFile
+ * @param {string} systemFile - System file name
+ * @param {string} appName - Application name
+ * @param {string} schemasPath - Schemas path
+ * @returns {Promise<string>} Validated system file path
+ * @throws {Error} If file not found
+ */
+async function validateSingleSystemFile(systemFile, appName, schemasPath) {
+  // Try new naming first: <app-name>-deploy.json in same folder
+  const newSystemPath = getDeployJsonPath(appName, 'external', true);
+  if (fsSync.existsSync(newSystemPath)) {
+    return newSystemPath;
+  }
+
+  // Fall back to specified path
+  const systemPath = path.join(schemasPath, systemFile);
+  try {
+    await fs.access(systemPath);
+    return systemPath;
+  } catch {
+    throw new Error(`External system file not found: ${systemPath} (also checked: ${newSystemPath})`);
+  }
+}
+
+/**
+ * Validates system files
+ * @async
+ * @function validateSystemFiles
+ * @param {string[]} systemFiles - Array of system file names
+ * @param {string} appName - Application name
+ * @param {string} schemasPath - Schemas path
+ * @returns {Promise<string[]>} Array of validated system file paths
+ */
+async function validateSystemFiles(systemFiles, appName, schemasPath) {
+  const validatedFiles = [];
+  for (const systemFile of systemFiles) {
+    const validatedPath = await validateSingleSystemFile(systemFile, appName, schemasPath);
+    validatedFiles.push(validatedPath);
+  }
+  return validatedFiles;
+}
+
+/**
+ * Validates a single datasource file
+ * @async
+ * @function validateSingleDatasourceFile
+ * @param {string} datasourceFile - Datasource file name
+ * @param {string} appPath - Application path
+ * @param {string} schemasPath - Schemas path
+ * @returns {Promise<string>} Validated datasource file path
+ * @throws {Error} If file not found
+ */
+async function validateSingleDatasourceFile(datasourceFile, appPath, schemasPath) {
+  // Try same folder first (new structure)
+  const datasourcePath = path.join(appPath, datasourceFile);
+  try {
+    await fs.access(datasourcePath);
+    return datasourcePath;
+  } catch {
+    // Fall back to schemaBasePath
+    const fallbackPath = path.join(schemasPath, datasourceFile);
+    try {
+      await fs.access(fallbackPath);
+      return fallbackPath;
+    } catch {
+      throw new Error(`External datasource file not found: ${datasourcePath} or ${fallbackPath}`);
+    }
+  }
+}
+
+/**
+ * Validates datasource files
+ * @async
+ * @function validateDatasourceFiles
+ * @param {string[]} datasourceFiles - Array of datasource file names
+ * @param {string} appPath - Application path
+ * @param {string} schemasPath - Schemas path
+ * @returns {Promise<string[]>} Array of validated datasource file paths
+ */
+async function validateDatasourceFiles(datasourceFiles, appPath, schemasPath) {
+  const validatedFiles = [];
+  for (const datasourceFile of datasourceFiles) {
+    const validatedPath = await validateSingleDatasourceFile(datasourceFile, appPath, schemasPath);
+    validatedFiles.push(validatedPath);
+  }
+  return validatedFiles;
+}
+
+/**
+ * Extracts system key from system file path
+ * @function extractSystemKey
+ * @param {string} systemFilePath - System file path
+ * @returns {string} System key
+ */
+function extractSystemKey(systemFilePath) {
+  // Normalize path separators first (handles Windows backslashes)
+  const normalizedPath = systemFilePath.replace(/\\/g, '/');
+  const systemFileName = path.basename(normalizedPath, '.json');
+  return systemFileName.replace(/-deploy$/, '');
+}
+
+module.exports = {
+  loadVariablesYaml,
+  validateSingleSystemFile,
+  validateSystemFiles,
+  validateSingleDatasourceFile,
+  validateDatasourceFiles,
+  extractSystemKey
+};
+

package/lib/{external-system-deploy.js → external-system/deploy.js}

@@ -10,9 +10,7 @@
  */
 
 const fs = require('fs').promises;
-const fsSync = require('fs');
 const path = require('path');
-const yaml = require('js-yaml');
 const chalk = require('chalk');
 const {
   deployExternalSystemViaPipeline,
@@ -20,13 +18,19 @@ const {
   uploadApplicationViaPipeline,
   validateUploadViaPipeline,
   publishUploadViaPipeline
-} = require('./api/pipeline.api');
-const { getDeploymentAuth } = require('./utils/token-manager');
-const { getConfig } = require('./config');
-const logger = require('./utils/logger');
-const { getDataplaneUrl } = require('./datasource-deploy');
-const { detectAppType, getDeployJsonPath } = require('./utils/paths');
-const { generateExternalSystemApplicationSchema } = require('./generator');
+} = require('../api/pipeline.api');
+const { getDeploymentAuth } = require('../utils/token-manager');
+const { getConfig } = require('../core/config');
+const logger = require('../utils/logger');
+const { getDataplaneUrl } = require('../datasource/deploy');
+const { detectAppType } = require('../utils/paths');
+const { generateExternalSystemApplicationSchema } = require('../generator/external');
+const {
+  loadVariablesYaml,
+  validateSystemFiles,
+  validateDatasourceFiles,
+  extractSystemKey
+} = require('./deploy-helpers');
 
 /**
  * Loads variables.yaml for an application
@@ -36,14 +40,6 @@ const { generateExternalSystemApplicationSchema } = require('./generator');
  * @returns {Promise<Object>} Variables configuration
  * @throws {Error} If file cannot be loaded
  */
-async function loadVariablesYaml(appName) {
-  // Detect app type and get correct path (integration or builder)
-  const { appPath } = await detectAppType(appName);
-  const variablesPath = path.join(appPath, 'variables.yaml');
-  const content = await fs.readFile(variablesPath, 'utf8');
-  return yaml.load(content);
-}
-
 /**
  * Validates external system files exist
  * @async
@@ -52,6 +48,7 @@ async function loadVariablesYaml(appName) {
  * @returns {Promise<Object>} Validation result with file paths
  * @throws {Error} If validation fails
  */
+
 async function validateExternalSystemFiles(appName) {
   const variables = await loadVariablesYaml(appName);
 
@@ -70,53 +67,18 @@ async function validateExternalSystemFiles(appName) {
     : path.join(appPath, schemaBasePath);
 
   // Validate system files
-  const systemFiles = [];
-  if (variables.externalIntegration.systems && variables.externalIntegration.systems.length > 0) {
-    for (const systemFile of variables.externalIntegration.systems) {
-      // Try new naming first: <app-name>-deploy.json in same folder
-      const newSystemPath = getDeployJsonPath(appName, 'external', true);
-      if (fsSync.existsSync(newSystemPath)) {
-        systemFiles.push(newSystemPath);
-      } else {
-        // Fall back to specified path
-        const systemPath = path.join(schemasPath, systemFile);
-        try {
-          await fs.access(systemPath);
-          systemFiles.push(systemPath);
-        } catch {
-          throw new Error(`External system file not found: ${systemPath} (also checked: ${newSystemPath})`);
-        }
-      }
-    }
-  } else {
+  const systemFilesList = variables.externalIntegration.systems || [];
+  if (systemFilesList.length === 0) {
     throw new Error('No external system files specified in externalIntegration.systems');
   }
+  const systemFiles = await validateSystemFiles(systemFilesList, appName, schemasPath);
 
-  // Validate datasource files (naming: <app-name>-deploy-<datasource-key>.json)
-  const datasourceFiles = [];
-  if (variables.externalIntegration.dataSources && variables.externalIntegration.dataSources.length > 0) {
-    for (const datasourceFile of variables.externalIntegration.dataSources) {
-      // Try same folder first (new structure)
-      const datasourcePath = path.join(appPath, datasourceFile);
-      try {
-        await fs.access(datasourcePath);
-        datasourceFiles.push(datasourcePath);
-      } catch {
-        // Fall back to schemaBasePath
-        const fallbackPath = path.join(schemasPath, datasourceFile);
-        try {
-          await fs.access(fallbackPath);
-          datasourceFiles.push(fallbackPath);
-        } catch {
-          throw new Error(`External datasource file not found: ${datasourcePath} or ${fallbackPath}`);
-        }
-      }
-    }
-  }
+  // Validate datasource files
+  const datasourceFilesList = variables.externalIntegration.dataSources || [];
+  const datasourceFiles = await validateDatasourceFiles(datasourceFilesList, appPath, schemasPath);
 
-  // Extract systemKey from system file (remove -deploy.json suffix if present)
-  const systemFileName = path.basename(systemFiles[0], '.json');
-  const systemKey = systemFileName.replace(/-deploy$/, '');
+  // Extract systemKey from system file
+  const systemKey = extractSystemKey(systemFiles[0]);
 
   return {
     systemFiles,
@@ -134,57 +96,121 @@ async function validateExternalSystemFiles(appName) {
  * @returns {Promise<void>} Resolves when deployment completes
  * @throws {Error} If deployment fails
  */
-async function buildExternalSystem(appName, options = {}) {
-  try {
-    logger.log(chalk.blue(`\n🔨 Building external system: ${appName}`));
+/**
+ * Validates and prepares deployment configuration
+ * @async
+ * @function prepareDeploymentConfig
+ * @param {string} appName - Application name
+ * @param {Object} options - Deployment options
+ * @returns {Promise<Object>} Deployment configuration
+ */
+async function prepareDeploymentConfig(appName, options) {
+  const { systemFiles, datasourceFiles, systemKey } = await validateExternalSystemFiles(appName);
 
-    // Validate files
-    const { systemFiles, datasourceFiles, systemKey } = await validateExternalSystemFiles(appName);
+  const config = await getConfig();
+  const environment = options.environment || 'dev';
+  const controllerUrl = options.controller || config.deployment?.controllerUrl || 'http://localhost:3000';
+  const authConfig = await getDeploymentAuth(controllerUrl, environment, appName);
 
-    // Get authentication
-    const config = await getConfig();
-    const environment = options.environment || 'dev';
-    const controllerUrl = options.controller || config.deployment?.controllerUrl || 'http://localhost:3000';
-    const authConfig = await getDeploymentAuth(controllerUrl, environment, appName);
+  if (!authConfig.token && !authConfig.clientId) {
+    throw new Error('Authentication required. Run "aifabrix login" or "aifabrix app register" first.');
+  }
 
-    if (!authConfig.token && !authConfig.clientId) {
-      throw new Error('Authentication required. Run "aifabrix login" or "aifabrix app register" first.');
-    }
+  return { systemFiles, datasourceFiles, systemKey, environment, controllerUrl, authConfig };
+}
 
-    // Get dataplane URL from controller
-    logger.log(chalk.blue('🌐 Getting dataplane URL from controller...'));
-    const dataplaneUrl = await getDataplaneUrl(controllerUrl, appName, environment, authConfig);
-    logger.log(chalk.green(`✓ Dataplane URL: ${dataplaneUrl}`));
+/**
+ * Gets dataplane URL from controller
+ * @async
+ * @function getDataplaneUrlForDeployment
+ * @param {string} controllerUrl - Controller URL
+ * @param {string} appName - Application name
+ * @param {string} environment - Environment key
+ * @param {Object} authConfig - Authentication configuration
+ * @returns {Promise<string>} Dataplane URL
+ */
+async function getDataplaneUrlForDeployment(controllerUrl, appName, environment, authConfig) {
+  logger.log(chalk.blue('🌐 Getting dataplane URL from controller...'));
+  const dataplaneUrl = await getDataplaneUrl(controllerUrl, appName, environment, authConfig);
+  logger.log(chalk.green(`✓ Dataplane URL: ${dataplaneUrl}`));
+  return dataplaneUrl;
+}
 
-    // Deploy external system
-    logger.log(chalk.blue(`Deploying external system: ${systemKey}...`));
-    const systemContent = await fs.readFile(systemFiles[0], 'utf8');
-    const systemJson = JSON.parse(systemContent);
+/**
+ * Deploys external system via pipeline
+ * @async
+ * @function deploySystem
+ * @param {string} dataplaneUrl - Dataplane URL
+ * @param {Object} authConfig - Authentication configuration
+ * @param {string} systemFilePath - Path to system file
+ * @param {string} systemKey - System key
+ * @returns {Promise<void>}
+ */
+async function deploySystem(dataplaneUrl, authConfig, systemFilePath, systemKey) {
+  logger.log(chalk.blue(`Deploying external system: ${systemKey}...`));
+  const systemContent = await fs.readFile(systemFilePath, 'utf8');
+  const systemJson = JSON.parse(systemContent);
 
-    const systemResponse = await deployExternalSystemViaPipeline(dataplaneUrl, authConfig, systemJson);
+  const systemResponse = await deployExternalSystemViaPipeline(dataplaneUrl, authConfig, systemJson);
 
-    if (!systemResponse.success) {
-      throw new Error(`Failed to deploy external system: ${systemResponse.error || systemResponse.formattedError}`);
-    }
+  if (!systemResponse.success) {
+    throw new Error(`Failed to deploy external system: ${systemResponse.error || systemResponse.formattedError}`);
+  }
 
-    logger.log(chalk.green(`✓ External system deployed: ${systemKey}`));
+  logger.log(chalk.green(`✓ External system deployed: ${systemKey}`));
+}
 
-    // Deploy datasources
-    for (const datasourceFile of datasourceFiles) {
-      const datasourceName = path.basename(datasourceFile, '.json');
-      logger.log(chalk.blue(`Deploying datasource: ${datasourceName}...`));
+/**
+ * Deploys a single datasource via pipeline
+ * @async
+ * @function deploySingleDatasource
+ * @param {string} dataplaneUrl - Dataplane URL
+ * @param {string} systemKey - System key
+ * @param {Object} authConfig - Authentication configuration
+ * @param {string} datasourceFile - Path to datasource file
+ * @returns {Promise<void>}
+ */
+async function deploySingleDatasource(dataplaneUrl, systemKey, authConfig, datasourceFile) {
+  const datasourceName = path.basename(datasourceFile, '.json');
+  logger.log(chalk.blue(`Deploying datasource: ${datasourceName}...`));
 
-      const datasourceContent = await fs.readFile(datasourceFile, 'utf8');
-      const datasourceJson = JSON.parse(datasourceContent);
+  const datasourceContent = await fs.readFile(datasourceFile, 'utf8');
+  const datasourceJson = JSON.parse(datasourceContent);
 
-      const datasourceResponse = await deployDatasourceViaPipeline(dataplaneUrl, systemKey, authConfig, datasourceJson);
+  const datasourceResponse = await deployDatasourceViaPipeline(dataplaneUrl, systemKey, authConfig, datasourceJson);
 
-      if (!datasourceResponse.success) {
-        throw new Error(`Failed to deploy datasource ${datasourceName}: ${datasourceResponse.error || datasourceResponse.formattedError}`);
-      }
+  if (!datasourceResponse.success) {
+    throw new Error(`Failed to deploy datasource ${datasourceName}: ${datasourceResponse.error || datasourceResponse.formattedError}`);
+  }
 
-      logger.log(chalk.green(`✓ Datasource deployed: ${datasourceName}`));
-    }
+  logger.log(chalk.green(`✓ Datasource deployed: ${datasourceName}`));
+}
+
+/**
+ * Deploys all datasources
+ * @async
+ * @function deployAllDatasources
+ * @param {string} dataplaneUrl - Dataplane URL
+ * @param {string} systemKey - System key
+ * @param {Object} authConfig - Authentication configuration
+ * @param {string[]} datasourceFiles - Array of datasource file paths
+ * @returns {Promise<void>}
+ */
+async function deployAllDatasources(dataplaneUrl, systemKey, authConfig, datasourceFiles) {
+  for (const datasourceFile of datasourceFiles) {
+    await deploySingleDatasource(dataplaneUrl, systemKey, authConfig, datasourceFile);
+  }
+}
+
+async function buildExternalSystem(appName, options = {}) {
+  try {
+    logger.log(chalk.blue(`\n🔨 Building external system: ${appName}`));
+
+    const { systemFiles, datasourceFiles, systemKey, environment, controllerUrl, authConfig } = await prepareDeploymentConfig(appName, options);
+    const dataplaneUrl = await getDataplaneUrlForDeployment(controllerUrl, appName, environment, authConfig);
+
+    await deploySystem(dataplaneUrl, authConfig, systemFiles[0], systemKey);
+    await deployAllDatasources(dataplaneUrl, systemKey, authConfig, datasourceFiles);
 
     logger.log(chalk.green('\n✅ External system built successfully!'));
     logger.log(chalk.blue(`System: ${systemKey}`));
@@ -268,26 +294,45 @@ async function uploadApplication(dataplaneUrl, authConfig, applicationSchema) {
  * @returns {Promise<void>}
  * @throws {Error} If validation fails
  */
-async function validateUpload(dataplaneUrl, uploadId, authConfig) {
-  logger.log(chalk.blue('🔍 Validating upload...'));
-  const validateResponse = await validateUploadViaPipeline(dataplaneUrl, uploadId, authConfig);
-
-  if (!validateResponse.success || !validateResponse.data) {
-    throw new Error(`Validation failed: ${validateResponse.error || validateResponse.formattedError || 'Unknown error'}`);
-  }
-
-  const validateData = validateResponse.data.data || validateResponse.data;
-
-  // Display changes
-  if (validateData.changes && validateData.changes.length > 0) {
+/**
+ * Displays validation changes
+ * @function displayValidationChanges
+ * @param {Object[]} changes - Array of changes
+ */
+function displayValidationChanges(changes) {
+  if (changes && changes.length > 0) {
     logger.log(chalk.blue('\n📋 Changes to be published:'));
-    for (const change of validateData.changes) {
+    for (const change of changes) {
       const changeType = change.type || 'unknown';
       const changeEntity = change.entity || change.key || 'unknown';
       const emoji = changeType === 'new' ? '➕' : changeType === 'modified' ? '✏️' : '🗑️';
       logger.log(chalk.gray(`  ${emoji} ${changeType}: ${changeEntity}`));
     }
   }
+}
+
+/**
+ * Validates upload response
+ * @function validateUploadResponse
+ * @param {Object} validateResponse - Validation response
+ * @returns {Object} Validation data
+ * @throws {Error} If validation failed
+ */
+function validateUploadResponse(validateResponse) {
+  if (!validateResponse.success || !validateResponse.data) {
+    throw new Error(`Validation failed: ${validateResponse.error || validateResponse.formattedError || 'Unknown error'}`);
+  }
+
+  return validateResponse.data.data || validateResponse.data;
+}
+
+async function validateUpload(dataplaneUrl, uploadId, authConfig) {
+  logger.log(chalk.blue('🔍 Validating upload...'));
+  const validateResponse = await validateUploadViaPipeline(dataplaneUrl, uploadId, authConfig);
+
+  const validateData = validateUploadResponse(validateResponse);
+
+  displayValidationChanges(validateData.changes);
 
   if (validateData.summary) {
     logger.log(chalk.blue(`\n📊 Summary: ${validateData.summary}`));

package/lib/external-system/download-helpers.js

@@ -0,0 +1,114 @@
+/**
+ * External System Download Helpers
+ *
+ * Helper functions for external system download file generation
+ *
+ * @fileoverview Download helper utilities for external system download
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+/**
+ * Generates variables.yaml content for downloaded system
+ * @param {string} systemKey - System key
+ * @param {Object} application - External system configuration
+ * @param {Array} dataSources - Array of datasource configurations
+ * @returns {Object} Variables YAML object
+ */
+function generateVariablesYaml(systemKey, application, dataSources) {
+  const systemFileName = `${systemKey}-deploy.json`;
+  const datasourceFiles = dataSources.map(ds => {
+    // Extract entity type from datasource key or use entityType/entityKey (backward compatibility)
+    const entityType = ds.entityType || ds.entityKey || ds.key.split('-').pop();
+    return `${systemKey}-deploy-${entityType}.json`;
+  });
+
+  return {
+    name: systemKey,
+    displayName: application.displayName || systemKey,
+    description: application.description || `External system integration for ${systemKey}`,
+    externalIntegration: {
+      schemaBasePath: './',
+      systems: [systemFileName],
+      dataSources: datasourceFiles,
+      autopublish: false,
+      version: application.version || '1.0.0'
+    }
+  };
+}
+
+/**
+ * Generates README.md with setup instructions
+ * @param {string} systemKey - System key
+ * @param {Object} application - External system configuration
+ * @param {Array} dataSources - Array of datasource configurations
+ * @returns {string} README.md content
+ */
+function generateReadme(systemKey, application, dataSources) {
+  const displayName = application.displayName || systemKey;
+  const description = application.description || `External system integration for ${systemKey}`;
+  const systemType = application.type || 'unknown';
+
+  const lines = [
+    `# ${displayName}`,
+    '',
+    description,
+    '',
+    '## System Information',
+    '',
+    `- **System Key**: \`${systemKey}\``,
+    `- **System Type**: \`${systemType}\``,
+    `- **Datasources**: ${dataSources.length}`,
+    '',
+    '## Files',
+    '',
+    '- `variables.yaml` - Application configuration with externalIntegration block',
+    `- \`${systemKey}-deploy.json\` - External system definition`
+  ];
+
+  dataSources.forEach(ds => {
+    const entityType = ds.entityType || ds.entityKey || ds.key.split('-').pop();
+    lines.push(`- \`${systemKey}-deploy-${entityType}.json\` - Datasource: ${ds.displayName || ds.key}`);
+  });
+
+  lines.push(
+    '- `env.template` - Environment variables template',
+    '',
+    '## Setup Instructions',
+    '',
+    '1. Review and update configuration files as needed',
+    '2. Set up environment variables in `env.template`',
+    '3. Run unit tests: `aifabrix test ${systemKey}`',
+    '4. Run integration tests: `aifabrix test-integration ${systemKey}`',
+    '5. Deploy: `aifabrix deploy ${systemKey} --environment dev`',
+    '',
+    '## Testing',
+    '',
+    '### Unit Tests',
+    'Run local validation without API calls:',
+    '```bash',
+    `aifabrix test ${systemKey}`,
+    '```',
+    '',
+    '### Integration Tests',
+    'Run integration tests via dataplane:',
+    '```bash',
+    `aifabrix test-integration ${systemKey} --environment dev`,
+    '```',
+    '',
+    '## Deployment',
+    '',
+    'Deploy to dataplane via miso-controller:',
+    '```bash',
+    `aifabrix deploy ${systemKey} --environment dev`,
+    '```'
+  );
+
+  return lines.join('\n');
+}
+
+module.exports = {
+  generateVariablesYaml,
+  generateReadme
+};
+