@aifabrix/builder 2.7.0 → 2.9.0

package/lib/environment-deploy.js

@@ -0,0 +1,305 @@
+/**
+ * AI Fabrix Builder Environment Deployment Module
+ *
+ * Handles environment deployment/setup in Miso Controller.
+ * Sets up environment infrastructure before applications can be deployed.
+ *
+ * @fileoverview Environment deployment for AI Fabrix Builder
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const chalk = require('chalk');
+const logger = require('./utils/logger');
+const config = require('./config');
+const { validateControllerUrl, validateEnvironmentKey } = require('./utils/deployment-validation');
+const { getOrRefreshDeviceToken } = require('./utils/token-manager');
+const { authenticatedApiCall } = require('./utils/api');
+const { handleDeploymentErrors } = require('./utils/deployment-errors');
+const auditLogger = require('./audit-logger');
+
+/**
+ * Validates environment deployment prerequisites
+ * @param {string} envKey - Environment key
+ * @param {string} controllerUrl - Controller URL
+ * @throws {Error} If prerequisites are not met
+ */
+function validateEnvironmentPrerequisites(envKey, controllerUrl) {
+  if (!envKey || typeof envKey !== 'string') {
+    throw new Error('Environment key is required');
+  }
+
+  if (!controllerUrl || typeof controllerUrl !== 'string') {
+    throw new Error('Controller URL is required');
+  }
+
+  // Validate environment key format
+  validateEnvironmentKey(envKey);
+
+  // Validate controller URL
+  validateControllerUrl(controllerUrl);
+}
+
+/**
+ * Gets authentication for environment deployment
+ * Uses device token (not app-specific client credentials)
+ * @async
+ * @param {string} controllerUrl - Controller URL
+ * @returns {Promise<Object>} Authentication configuration
+ * @throws {Error} If authentication is not available
+ */
+async function getEnvironmentAuth(controllerUrl) {
+  const validatedUrl = validateControllerUrl(controllerUrl);
+
+  // Get or refresh device token
+  const deviceToken = await getOrRefreshDeviceToken(validatedUrl);
+
+  if (!deviceToken || !deviceToken.token) {
+    throw new Error('Device token is required for environment deployment. Run "aifabrix login" first to authenticate.');
+  }
+
+  return {
+    type: 'device',
+    token: deviceToken.token,
+    controller: deviceToken.controller || validatedUrl
+  };
+}
+
+/**
+ * Sends environment deployment request to controller
+ * @async
+ * @param {string} controllerUrl - Controller URL
+ * @param {string} envKey - Environment key
+ * @param {Object} authConfig - Authentication configuration
+ * @param {Object} options - Deployment options
+ * @returns {Promise<Object>} Deployment result
+ * @throws {Error} If deployment fails
+ */
+async function sendEnvironmentDeployment(controllerUrl, envKey, authConfig, options = {}) {
+  const validatedUrl = validateControllerUrl(controllerUrl);
+  const validatedEnvKey = validateEnvironmentKey(envKey);
+
+  // Build environment deployment request
+  const deploymentRequest = {
+    key: validatedEnvKey,
+    displayName: `${validatedEnvKey.charAt(0).toUpperCase() + validatedEnvKey.slice(1)} Environment`,
+    description: `${validatedEnvKey.charAt(0).toUpperCase() + validatedEnvKey.slice(1)} environment for application deployments`
+  };
+
+  // Add configuration if provided
+  if (options.config) {
+    // TODO: Load and parse config file if provided
+    // For now, just include the config path in description
+    deploymentRequest.description += ` (config: ${options.config})`;
+  }
+
+  // API endpoint: POST /api/v1/environments/{env}/deploy
+  // Alternative: POST /api/v1/environments/deploy with environment in body
+  const endpoint = `${validatedUrl}/api/v1/environments/${validatedEnvKey}/deploy`;
+
+  // Log deployment attempt for audit
+  await auditLogger.logDeploymentAttempt(validatedEnvKey, validatedUrl, options);
+
+  try {
+    const response = await authenticatedApiCall(
+      endpoint,
+      {
+        method: 'POST',
+        body: JSON.stringify(deploymentRequest)
+      },
+      authConfig.token
+    );
+
+    if (!response.success) {
+      const error = new Error(response.formattedError || response.error || 'Environment deployment failed');
+      error.status = response.status;
+      error.data = response.errorData;
+      throw error;
+    }
+
+    // Handle response structure
+    const responseData = response.data || {};
+    return {
+      success: true,
+      environment: validatedEnvKey,
+      deploymentId: responseData.deploymentId || responseData.id,
+      status: responseData.status || 'initiated',
+      url: responseData.url || `${validatedUrl}/environments/${validatedEnvKey}`,
+      message: responseData.message
+    };
+  } catch (error) {
+    // Use unified error handler
+    await handleDeploymentErrors(error, validatedEnvKey, validatedUrl, false);
+    throw error;
+  }
+}
+
+/**
+ * Polls environment deployment status
+ * @async
+ * @param {string} deploymentId - Deployment ID
+ * @param {string} controllerUrl - Controller URL
+ * @param {string} envKey - Environment key
+ * @param {Object} authConfig - Authentication configuration
+ * @param {Object} options - Polling options
+ * @returns {Promise<Object>} Final deployment status
+ */
+async function pollEnvironmentStatus(deploymentId, controllerUrl, envKey, authConfig, options = {}) {
+  const validatedUrl = validateControllerUrl(controllerUrl);
+  const validatedEnvKey = validateEnvironmentKey(envKey);
+
+  const pollInterval = options.pollInterval || 5000;
+  const maxAttempts = options.maxAttempts || 60;
+  const statusEndpoint = `${validatedUrl}/api/v1/environments/${validatedEnvKey}/status`;
+
+  logger.log(chalk.blue(`⏳ Polling environment status (${pollInterval}ms intervals)...`));
+
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      await new Promise(resolve => setTimeout(resolve, pollInterval));
+
+      const response = await authenticatedApiCall(
+        statusEndpoint,
+        {
+          method: 'GET'
+        },
+        authConfig.token
+      );
+
+      if (response.success && response.data) {
+        const status = response.data.status || response.data.ready;
+        const isReady = status === 'ready' || status === 'completed' || response.data.ready === true;
+
+        if (isReady) {
+          return {
+            success: true,
+            environment: validatedEnvKey,
+            status: 'ready',
+            message: 'Environment is ready for application deployments'
+          };
+        }
+
+        // Check for terminal failure states
+        if (status === 'failed' || status === 'error') {
+          throw new Error(`Environment deployment failed: ${response.data.message || 'Unknown error'}`);
+        }
+      }
+    } catch (error) {
+      // If it's a terminal error (not a timeout), throw it
+      if (error.message && error.message.includes('failed')) {
+        throw error;
+      }
+      // Otherwise, continue polling
+    }
+
+    if (attempt < maxAttempts) {
+      logger.log(chalk.gray(`   Attempt ${attempt}/${maxAttempts}...`));
+    }
+  }
+
+  // Timeout
+  throw new Error(`Environment deployment timeout after ${maxAttempts} attempts. Check controller logs for status.`);
+}
+
+/**
+ * Displays environment deployment results
+ * @param {Object} result - Deployment result
+ */
+function displayDeploymentResults(result) {
+  logger.log(chalk.green('\n✅ Environment deployed successfully'));
+  logger.log(chalk.blue(`   Environment: ${result.environment}`));
+  logger.log(chalk.blue(`   Status: ${result.status === 'ready' ? '✅ ready' : result.status}`));
+  if (result.url) {
+    logger.log(chalk.blue(`   URL: ${result.url}`));
+  }
+  if (result.deploymentId) {
+    logger.log(chalk.blue(`   Deployment ID: ${result.deploymentId}`));
+  }
+  logger.log(chalk.green('\n✓ Environment is ready for application deployments'));
+}
+
+/**
+ * Deploys/setups an environment in the controller
+ * @async
+ * @function deployEnvironment
+ * @param {string} envKey - Environment key (miso, dev, tst, pro)
+ * @param {Object} options - Deployment options
+ * @param {string} options.controller - Controller URL (required)
+ * @param {string} [options.config] - Environment configuration file (optional)
+ * @param {boolean} [options.skipValidation] - Skip validation checks
+ * @param {boolean} [options.poll] - Poll for deployment status (default: true)
+ * @param {boolean} [options.noPoll] - Do not poll for status
+ * @returns {Promise<Object>} Deployment result
+ * @throws {Error} If deployment fails
+ *
+ * @example
+ * await deployEnvironment('dev', { controller: 'https://controller.aifabrix.ai' });
+ */
+async function deployEnvironment(envKey, options = {}) {
+  try {
+    // 1. Input validation
+    if (!envKey || typeof envKey !== 'string' || envKey.trim().length === 0) {
+      throw new Error('Environment key is required');
+    }
+
+    const controllerUrl = options.controller || options['controller-url'];
+    if (!controllerUrl) {
+      throw new Error('Controller URL is required. Use --controller flag');
+    }
+
+    // 2. Validate prerequisites
+    if (!options.skipValidation) {
+      validateEnvironmentPrerequisites(envKey, controllerUrl);
+    }
+
+    // 3. Update root-level environment in config.yaml
+    await config.setCurrentEnvironment(envKey);
+
+    // 4. Get authentication (device token)
+    logger.log(chalk.blue(`\n📋 Deploying environment '${envKey}' to ${controllerUrl}...`));
+    const authConfig = await getEnvironmentAuth(controllerUrl);
+    logger.log(chalk.green('✓ Environment validated'));
+    logger.log(chalk.green('✓ Authentication successful'));
+
+    // 5. Send environment deployment request
+    logger.log(chalk.blue('\n🚀 Deploying environment infrastructure...'));
+    const validatedControllerUrl = validateControllerUrl(authConfig.controller);
+    const result = await sendEnvironmentDeployment(validatedControllerUrl, envKey, authConfig, options);
+
+    logger.log(chalk.blue(`📤 Sending deployment request to ${validatedControllerUrl}/api/v1/environments/${envKey}/deploy...`));
+
+    // 6. Poll for status if enabled
+    const shouldPoll = options.poll !== false && !options.noPoll;
+    if (shouldPoll && result.deploymentId) {
+      const pollResult = await pollEnvironmentStatus(
+        result.deploymentId,
+        validatedControllerUrl,
+        envKey,
+        authConfig,
+        {
+          pollInterval: 5000,
+          maxAttempts: 60
+        }
+      );
+      result.status = pollResult.status;
+      result.message = pollResult.message;
+    }
+
+    // 7. Display results
+    displayDeploymentResults(result);
+
+    return result;
+  } catch (error) {
+    // Error handling is done in sendEnvironmentDeployment and pollEnvironmentStatus
+    // Re-throw with context
+    if (error._logged !== true) {
+      logger.error(chalk.red(`\n❌ Environment deployment failed: ${error.message}`));
+    }
+    throw error;
+  }
+}
+
+module.exports = {
+  deployEnvironment
+};
+

package/lib/external-system-deploy.js

@@ -0,0 +1,345 @@
+/**
+ * External System Deployment Module
+ *
+ * Handles deployment of external systems and datasources via pipeline API
+ * for external type applications.
+ *
+ * @fileoverview External system deployment for AI Fabrix Builder
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fs = require('fs').promises;
+const fsSync = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+const chalk = require('chalk');
+const { authenticatedApiCall } = require('./utils/api');
+const { getDeploymentAuth } = require('./utils/token-manager');
+const { getConfig } = require('./config');
+const logger = require('./utils/logger');
+const { getDataplaneUrl } = require('./datasource-deploy');
+const { detectAppType, getDeployJsonPath } = require('./utils/paths');
+const { generateExternalSystemApplicationSchema } = require('./generator');
+
+/**
+ * Loads variables.yaml for an application
+ * @async
+ * @function loadVariablesYaml
+ * @param {string} appName - Application name
+ * @returns {Promise<Object>} Variables configuration
+ * @throws {Error} If file cannot be loaded
+ */
+async function loadVariablesYaml(appName) {
+  // Detect app type and get correct path (integration or builder)
+  const { appPath } = await detectAppType(appName);
+  const variablesPath = path.join(appPath, 'variables.yaml');
+  const content = await fs.readFile(variablesPath, 'utf8');
+  return yaml.load(content);
+}
+
+/**
+ * Validates external system files exist
+ * @async
+ * @function validateExternalSystemFiles
+ * @param {string} appName - Application name
+ * @returns {Promise<Object>} Validation result with file paths
+ * @throws {Error} If validation fails
+ */
+async function validateExternalSystemFiles(appName) {
+  const variables = await loadVariablesYaml(appName);
+
+  if (!variables.externalIntegration) {
+    throw new Error('externalIntegration block not found in variables.yaml');
+  }
+
+  // Detect app type and get correct path (integration or builder)
+  const { appPath } = await detectAppType(appName);
+
+  // For new structure, files are in same folder (schemaBasePath is usually './')
+  // For backward compatibility, support old schemas/ subfolder
+  const schemaBasePath = variables.externalIntegration.schemaBasePath || './';
+  const schemasPath = path.isAbsolute(schemaBasePath)
+    ? schemaBasePath
+    : path.join(appPath, schemaBasePath);
+
+  // Validate system files
+  const systemFiles = [];
+  if (variables.externalIntegration.systems && variables.externalIntegration.systems.length > 0) {
+    for (const systemFile of variables.externalIntegration.systems) {
+      // Try new naming first: <app-name>-deploy.json in same folder
+      const newSystemPath = getDeployJsonPath(appName, 'external', true);
+      if (fsSync.existsSync(newSystemPath)) {
+        systemFiles.push(newSystemPath);
+      } else {
+        // Fall back to specified path
+        const systemPath = path.join(schemasPath, systemFile);
+        try {
+          await fs.access(systemPath);
+          systemFiles.push(systemPath);
+        } catch {
+          throw new Error(`External system file not found: ${systemPath} (also checked: ${newSystemPath})`);
+        }
+      }
+    }
+  } else {
+    throw new Error('No external system files specified in externalIntegration.systems');
+  }
+
+  // Validate datasource files (naming: <app-name>-deploy-<datasource-key>.json)
+  const datasourceFiles = [];
+  if (variables.externalIntegration.dataSources && variables.externalIntegration.dataSources.length > 0) {
+    for (const datasourceFile of variables.externalIntegration.dataSources) {
+      // Try same folder first (new structure)
+      const datasourcePath = path.join(appPath, datasourceFile);
+      try {
+        await fs.access(datasourcePath);
+        datasourceFiles.push(datasourcePath);
+      } catch {
+        // Fall back to schemaBasePath
+        const fallbackPath = path.join(schemasPath, datasourceFile);
+        try {
+          await fs.access(fallbackPath);
+          datasourceFiles.push(fallbackPath);
+        } catch {
+          throw new Error(`External datasource file not found: ${datasourcePath} or ${fallbackPath}`);
+        }
+      }
+    }
+  }
+
+  // Extract systemKey from system file (remove -deploy.json suffix if present)
+  const systemFileName = path.basename(systemFiles[0], '.json');
+  const systemKey = systemFileName.replace(/-deploy$/, '');
+
+  return {
+    systemFiles,
+    datasourceFiles,
+    systemKey
+  };
+}
+
+/**
+ * Deploys external system to dataplane (build step - deploy, not publish)
+ * @async
+ * @function buildExternalSystem
+ * @param {string} appName - Application name
+ * @param {Object} options - Deployment options
+ * @returns {Promise<void>} Resolves when deployment completes
+ * @throws {Error} If deployment fails
+ */
+async function buildExternalSystem(appName, options = {}) {
+  try {
+    logger.log(chalk.blue(`\n🔨 Building external system: ${appName}`));
+
+    // Validate files
+    const { systemFiles, datasourceFiles, systemKey } = await validateExternalSystemFiles(appName);
+
+    // Get authentication
+    const config = await getConfig();
+    const environment = options.environment || 'dev';
+    const controllerUrl = options.controller || config.deployment?.controllerUrl || 'http://localhost:3000';
+    const authConfig = await getDeploymentAuth(controllerUrl, environment, appName);
+
+    if (!authConfig.token && !authConfig.clientId) {
+      throw new Error('Authentication required. Run "aifabrix login" or "aifabrix app register" first.');
+    }
+
+    // Get dataplane URL from controller
+    logger.log(chalk.blue('🌐 Getting dataplane URL from controller...'));
+    const dataplaneUrl = await getDataplaneUrl(controllerUrl, appName, environment, authConfig);
+    logger.log(chalk.green(`✓ Dataplane URL: ${dataplaneUrl}`));
+
+    // Deploy external system
+    logger.log(chalk.blue(`Deploying external system: ${systemKey}...`));
+    const systemContent = await fs.readFile(systemFiles[0], 'utf8');
+    const systemJson = JSON.parse(systemContent);
+
+    const systemResponse = await authenticatedApiCall(
+      `${dataplaneUrl}/api/v1/pipeline/deploy`,
+      {
+        method: 'POST',
+        body: JSON.stringify(systemJson)
+      },
+      authConfig.token
+    );
+
+    if (!systemResponse.success) {
+      throw new Error(`Failed to deploy external system: ${systemResponse.error || systemResponse.formattedError}`);
+    }
+
+    logger.log(chalk.green(`✓ External system deployed: ${systemKey}`));
+
+    // Deploy datasources
+    for (const datasourceFile of datasourceFiles) {
+      const datasourceName = path.basename(datasourceFile, '.json');
+      logger.log(chalk.blue(`Deploying datasource: ${datasourceName}...`));
+
+      const datasourceContent = await fs.readFile(datasourceFile, 'utf8');
+      const datasourceJson = JSON.parse(datasourceContent);
+
+      const datasourceResponse = await authenticatedApiCall(
+        `${dataplaneUrl}/api/v1/pipeline/${systemKey}/deploy`,
+        {
+          method: 'POST',
+          body: JSON.stringify(datasourceJson)
+        },
+        authConfig.token
+      );
+
+      if (!datasourceResponse.success) {
+        throw new Error(`Failed to deploy datasource ${datasourceName}: ${datasourceResponse.error || datasourceResponse.formattedError}`);
+      }
+
+      logger.log(chalk.green(`✓ Datasource deployed: ${datasourceName}`));
+    }
+
+    logger.log(chalk.green('\n✅ External system built successfully!'));
+    logger.log(chalk.blue(`System: ${systemKey}`));
+    logger.log(chalk.blue(`Datasources: ${datasourceFiles.length}`));
+  } catch (error) {
+    throw new Error(`Failed to build external system: ${error.message}`);
+  }
+}
+
+/**
+ * Publishes external system to dataplane using application-level workflow
+ * Uses upload → validate → publish workflow for atomic deployment
+ * @async
+ * @function deployExternalSystem
+ * @param {string} appName - Application name
+ * @param {Object} options - Deployment options
+ * @param {string} [options.environment] - Environment (dev, tst, pro)
+ * @param {string} [options.controller] - Controller URL
+ * @param {boolean} [options.skipValidation] - Skip validation step and go straight to publish
+ * @param {boolean} [options.generateMcpContract] - Generate MCP contract (default: true)
+ * @returns {Promise<void>} Resolves when deployment completes
+ * @throws {Error} If deployment fails
+ */
+async function deployExternalSystem(appName, options = {}) {
+  try {
+    logger.log(chalk.blue(`\n🚀 Publishing external system: ${appName}`));
+
+    // Validate files
+    const { systemFiles: _systemFiles, datasourceFiles, systemKey } = await validateExternalSystemFiles(appName);
+
+    // Generate application-schema.json structure
+    logger.log(chalk.blue('📋 Generating application schema...'));
+    const applicationSchema = await generateExternalSystemApplicationSchema(appName);
+    logger.log(chalk.green('✓ Application schema generated'));
+
+    // Get authentication
+    const config = await getConfig();
+    const environment = options.environment || 'dev';
+    const controllerUrl = options.controller || config.deployment?.controllerUrl || 'http://localhost:3000';
+    const authConfig = await getDeploymentAuth(controllerUrl, environment, appName);
+
+    if (!authConfig.token && !authConfig.clientId) {
+      throw new Error('Authentication required. Run "aifabrix login" or "aifabrix app register" first.');
+    }
+
+    // Get dataplane URL from controller
+    logger.log(chalk.blue('🌐 Getting dataplane URL from controller...'));
+    const dataplaneUrl = await getDataplaneUrl(controllerUrl, appName, environment, authConfig);
+    logger.log(chalk.green(`✓ Dataplane URL: ${dataplaneUrl}`));
+
+    // Step 1: Upload application
+    logger.log(chalk.blue('📤 Uploading application configuration...'));
+    const uploadResponse = await authenticatedApiCall(
+      `${dataplaneUrl}/api/v1/pipeline/upload`,
+      {
+        method: 'POST',
+        body: JSON.stringify(applicationSchema)
+      },
+      authConfig.token
+    );
+
+    if (!uploadResponse.success || !uploadResponse.data) {
+      throw new Error(`Failed to upload application: ${uploadResponse.error || uploadResponse.formattedError || 'Unknown error'}`);
+    }
+
+    const uploadData = uploadResponse.data.data || uploadResponse.data;
+    const uploadId = uploadData.uploadId || uploadData.id;
+
+    if (!uploadId) {
+      throw new Error('Upload ID not found in upload response');
+    }
+
+    logger.log(chalk.green(`✓ Upload successful (ID: ${uploadId})`));
+
+    // Step 2: Validate upload (optional, can be skipped)
+    if (!options.skipValidation) {
+      logger.log(chalk.blue('🔍 Validating upload...'));
+      const validateResponse = await authenticatedApiCall(
+        `${dataplaneUrl}/api/v1/pipeline/upload/${uploadId}/validate`,
+        {
+          method: 'POST'
+        },
+        authConfig.token
+      );
+
+      if (!validateResponse.success || !validateResponse.data) {
+        throw new Error(`Validation failed: ${validateResponse.error || validateResponse.formattedError || 'Unknown error'}`);
+      }
+
+      const validateData = validateResponse.data.data || validateResponse.data;
+
+      // Display changes
+      if (validateData.changes && validateData.changes.length > 0) {
+        logger.log(chalk.blue('\n📋 Changes to be published:'));
+        for (const change of validateData.changes) {
+          const changeType = change.type || 'unknown';
+          const changeEntity = change.entity || change.key || 'unknown';
+          const emoji = changeType === 'new' ? '➕' : changeType === 'modified' ? '✏️' : '🗑️';
+          logger.log(chalk.gray(`  ${emoji} ${changeType}: ${changeEntity}`));
+        }
+      }
+
+      if (validateData.summary) {
+        logger.log(chalk.blue(`\n📊 Summary: ${validateData.summary}`));
+      }
+
+      logger.log(chalk.green('✓ Validation successful'));
+    } else {
+      logger.log(chalk.yellow('⚠ Skipping validation step'));
+    }
+
+    // Step 3: Publish application
+    const generateMcpContract = options.generateMcpContract !== false; // Default to true
+    logger.log(chalk.blue(`📢 Publishing application (MCP contract: ${generateMcpContract ? 'enabled' : 'disabled'})...`));
+
+    const publishResponse = await authenticatedApiCall(
+      `${dataplaneUrl}/api/v1/pipeline/upload/${uploadId}/publish?generateMcpContract=${generateMcpContract}`,
+      {
+        method: 'POST'
+      },
+      authConfig.token
+    );
+
+    if (!publishResponse.success || !publishResponse.data) {
+      throw new Error(`Failed to publish application: ${publishResponse.error || publishResponse.formattedError || 'Unknown error'}`);
+    }
+
+    const publishData = publishResponse.data.data || publishResponse.data;
+
+    logger.log(chalk.green('\n✅ External system published successfully!'));
+    logger.log(chalk.blue(`System: ${systemKey}`));
+    if (publishData.systems && publishData.systems.length > 0) {
+      logger.log(chalk.blue(`Published systems: ${publishData.systems.length}`));
+    }
+    if (publishData.dataSources && publishData.dataSources.length > 0) {
+      logger.log(chalk.blue(`Published datasources: ${publishData.dataSources.length}`));
+    } else {
+      logger.log(chalk.blue(`Datasources: ${datasourceFiles.length}`));
+    }
+  } catch (error) {
+    throw new Error(`Failed to deploy external system: ${error.message}`);
+  }
+}
+
+module.exports = {
+  buildExternalSystem,
+  deployExternalSystem,
+  validateExternalSystemFiles
+};
+