@aifabrix/builder 2.22.1 → 2.31.0

package/lib/utils/external-system-display.js

@@ -54,11 +54,8 @@ function displayTestResults(results, verbose = false) {
 
         if (dsResult.metadataSchemaResults) {
           const ms = dsResult.metadataSchemaResults;
-          if (ms.valid) {
-            logger.log(chalk.gray('    Metadata schema: ✓ Valid'));
-          } else {
-            logger.log(chalk.red('    Metadata schema: ✗ Invalid'));
-          }
+          const statusMsg = ms.valid ? '    Metadata schema: ✓ Valid' : '    Metadata schema: ✗ Invalid';
+          logger.log(ms.valid ? chalk.gray(statusMsg) : chalk.red(statusMsg));
         }
       }
     }
@@ -110,20 +107,18 @@ function displayIntegrationTestResults(results, verbose = false) {
       }
     }
 
-    if (verbose) {
-      if (dsResult.validationResults) {
-        const vr = dsResult.validationResults;
-        if (vr.isValid) {
-          logger.log(chalk.gray('    Validation: ✓ Valid'));
-        } else {
-          logger.log(chalk.red('    Validation: ✗ Invalid'));
-          if (vr.errors && vr.errors.length > 0) {
-            vr.errors.forEach(err => logger.log(chalk.red(`      - ${err}`)));
-          }
-        }
-        if (vr.warnings && vr.warnings.length > 0) {
-          vr.warnings.forEach(warn => logger.log(chalk.yellow(`      ⚠ ${warn}`)));
-        }
+    if (verbose && dsResult.validationResults) {
+      const vr = dsResult.validationResults;
+      if (vr.isValid) {
+        logger.log(chalk.gray('    Validation: ✓ Valid'));
+      } else {
+        logger.log(chalk.red('    Validation: ✗ Invalid'));
+      }
+      if (vr.errors && vr.errors.length > 0) {
+        vr.errors.forEach(err => logger.log(chalk.red(`      - ${err}`)));
+      }
+      if (vr.warnings && vr.warnings.length > 0) {
+        vr.warnings.forEach(warn => logger.log(chalk.yellow(`      ⚠ ${warn}`)));
       }
 
       if (dsResult.fieldMappingResults) {

package/lib/utils/external-system-env-helpers.js

@@ -0,0 +1,107 @@
+/**
+ * External System Environment Variable Helpers
+ *
+ * Helper functions for extracting environment variables from external system configurations.
+ * Separated from external-system-download.js to maintain file size limits.
+ *
+ * @fileoverview External system environment variable extraction helpers
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+/**
+ * Extract OAuth2 environment variables
+ * @param {Object} oauth2 - OAuth2 configuration
+ * @param {string} systemKey - System key
+ * @param {Array<string>} lines - Lines array to append to
+ */
+function extractOAuth2EnvVars(oauth2, systemKey, lines) {
+  if (oauth2.clientId && oauth2.clientId.includes('{{')) {
+    const key = oauth2.clientId.replace(/[{}]/g, '').trim();
+    lines.push(`${key}=kv://secrets/${systemKey}/client-id`);
+  }
+  if (oauth2.clientSecret && oauth2.clientSecret.includes('{{')) {
+    const key = oauth2.clientSecret.replace(/[{}]/g, '').trim();
+    lines.push(`${key}=kv://secrets/${systemKey}/client-secret`);
+  }
+}
+
+/**
+ * Extract API Key environment variables
+ * @param {Object} apikey - API Key configuration
+ * @param {string} systemKey - System key
+ * @param {Array<string>} lines - Lines array to append to
+ */
+function extractApiKeyEnvVars(apikey, systemKey, lines) {
+  if (apikey.key && apikey.key.includes('{{')) {
+    const key = apikey.key.replace(/[{}]/g, '').trim();
+    lines.push(`${key}=kv://secrets/${systemKey}/api-key`);
+  }
+}
+
+/**
+ * Extract Basic Auth environment variables
+ * @param {Object} basic - Basic Auth configuration
+ * @param {string} systemKey - System key
+ * @param {Array<string>} lines - Lines array to append to
+ */
+function extractBasicAuthEnvVars(basic, systemKey, lines) {
+  if (basic.username && basic.username.includes('{{')) {
+    const key = basic.username.replace(/[{}]/g, '').trim();
+    lines.push(`${key}=kv://secrets/${systemKey}/username`);
+  }
+  if (basic.password && basic.password.includes('{{')) {
+    const key = basic.password.replace(/[{}]/g, '').trim();
+    lines.push(`${key}=kv://secrets/${systemKey}/password`);
+  }
+}
+
+/**
+ * Extract authentication environment variables
+ * @param {Object} auth - Authentication configuration
+ * @param {string} systemKey - System key
+ * @param {Array<string>} lines - Lines array to append to
+ */
+function extractAuthEnvVars(auth, systemKey, lines) {
+  // OAuth2 configuration
+  if (auth.type === 'oauth2' && auth.oauth2) {
+    extractOAuth2EnvVars(auth.oauth2, systemKey, lines);
+  }
+
+  // API Key configuration
+  if (auth.type === 'apikey' && auth.apikey) {
+    extractApiKeyEnvVars(auth.apikey, systemKey, lines);
+  }
+
+  // Basic Auth configuration
+  if (auth.type === 'basic' && auth.basic) {
+    extractBasicAuthEnvVars(auth.basic, systemKey, lines);
+  }
+}
+
+/**
+ * Extracts environment variables from authentication configuration
+ * @param {Object} application - External system configuration
+ * @returns {string} Environment variables template content
+ */
+function generateEnvTemplate(application) {
+  const lines = ['# Environment variables for external system'];
+  lines.push(`# System: ${application.key || 'unknown'}`);
+  lines.push('');
+
+  if (!application.authentication) {
+    return lines.join('\n');
+  }
+
+  extractAuthEnvVars(application.authentication, application.key, lines);
+  return lines.join('\n');
+}
+
+module.exports = {
+  extractOAuth2EnvVars,
+  extractApiKeyEnvVars,
+  extractBasicAuthEnvVars,
+  extractAuthEnvVars,
+  generateEnvTemplate
+};
+

package/lib/utils/external-system-test-helpers.js

@@ -0,0 +1,144 @@
+/**
+ * External System Test Helper Functions
+ *
+ * Helper functions for external system testing.
+ * Separated from external-system-test.js to maintain file size limits.
+ *
+ * @fileoverview External system test helper functions
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fs = require('fs').promises;
+const path = require('path');
+const { testDatasourceViaPipeline } = require('../api/pipeline.api');
+
+/**
+ * Retry API call with exponential backoff
+ * @async
+ * @param {Function} fn - Function to retry
+ * @param {number} maxRetries - Maximum number of retries
+ * @param {number} backoffMs - Initial backoff delay in milliseconds
+ * @returns {Promise<any>} Function result
+ * @throws {Error} Last error if all retries fail
+ */
+async function retryApiCall(fn, maxRetries = 3, backoffMs = 1000) {
+  let lastError;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      if (attempt < maxRetries) {
+        const delay = backoffMs * Math.pow(2, attempt);
+        await new Promise(resolve => setTimeout(resolve, delay));
+      }
+    }
+  }
+  throw lastError;
+}
+
+/**
+ * Calls pipeline test endpoint using centralized API client
+ * @async
+ * @param {Object} params - Function parameters
+ * @param {string} params.systemKey - System key
+ * @param {string} params.datasourceKey - Datasource key
+ * @param {Object} params.payloadTemplate - Test payload template
+ * @param {string} params.dataplaneUrl - Dataplane URL
+ * @param {Object} params.authConfig - Authentication configuration
+ * @param {number} [params.timeout] - Request timeout in milliseconds (default: 30000)
+ * @returns {Promise<Object>} Test response
+ */
+async function callPipelineTestEndpoint({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout = 30000 }) {
+  const response = await retryApiCall(async() => {
+    return await testDatasourceViaPipeline({
+      dataplaneUrl,
+      systemKey,
+      datasourceKey,
+      authConfig,
+      testData: { payloadTemplate },
+      options: { timeout }
+    });
+  });
+
+  if (!response.success || !response.data) {
+    throw new Error(`Test endpoint failed: ${response.error || response.formattedError || 'Unknown error'}`);
+  }
+
+  return response.data.data || response.data;
+}
+
+/**
+ * Load custom payload from file if provided
+ * @async
+ * @param {string} [payloadPath] - Path to custom payload file
+ * @returns {Promise<Object|null>} Custom payload or null
+ */
+async function loadCustomPayload(payloadPath) {
+  if (!payloadPath) {
+    return null;
+  }
+
+  const absolutePath = path.isAbsolute(payloadPath) ? payloadPath : path.join(process.cwd(), payloadPath);
+  const payloadContent = await fs.readFile(absolutePath, 'utf8');
+  return JSON.parse(payloadContent);
+}
+
+/**
+ * Determine payload template for a datasource
+ * @param {Object} datasource - Datasource configuration
+ * @param {string} datasourceKey - Datasource key
+ * @param {Object|null} customPayload - Custom payload if provided
+ * @returns {Object|null} Payload template or null
+ */
+function determinePayloadTemplate(datasource, datasourceKey, customPayload) {
+  if (customPayload) {
+    return customPayload;
+  }
+  if (datasource.testPayload && datasource.testPayload.payloadTemplate) {
+    return datasource.testPayload.payloadTemplate;
+  }
+  return null;
+}
+
+/**
+ * Test a single datasource via pipeline
+ * @async
+ * @param {Object} params - Test parameters
+ * @param {string} params.systemKey - System key
+ * @param {string} params.datasourceKey - Datasource key
+ * @param {Object} params.payloadTemplate - Payload template
+ * @param {string} params.dataplaneUrl - Dataplane URL
+ * @param {Object} params.authConfig - Authentication configuration
+ * @param {number} params.timeout - Request timeout
+ * @returns {Promise<Object>} Test result
+ */
+async function testSingleDatasource({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout }) {
+  const testResponse = await callPipelineTestEndpoint({
+    systemKey,
+    datasourceKey,
+    payloadTemplate,
+    dataplaneUrl,
+    authConfig,
+    timeout
+  });
+
+  return {
+    key: datasourceKey,
+    skipped: false,
+    success: testResponse.success !== false,
+    validationResults: testResponse.validationResults || {},
+    fieldMappingResults: testResponse.fieldMappingResults || {},
+    endpointTestResults: testResponse.endpointTestResults || {}
+  };
+}
+
+module.exports = {
+  retryApiCall,
+  callPipelineTestEndpoint,
+  loadCustomPayload,
+  determinePayloadTemplate,
+  testSingleDatasource
+};
+

package/lib/utils/health-check.js

@@ -99,15 +99,17 @@ async function getContainerPort(appName, debug = false) {
       }
       const { stdout: psOutput } = await execAsync(psCmd);
       const portMatch = psOutput.match(/:(\d+)->/);
-      if (portMatch) {
-        const port = parseInt(portMatch[1], 10);
-        if (!isNaN(port) && port > 0) {
-          if (debug) {
-            logger.log(chalk.gray(`[DEBUG] Detected port ${port} from docker ps`));
-          }
-          return port;
-        }
+      if (!portMatch) {
+        return null;
+      }
+      const port = parseInt(portMatch[1], 10);
+      if (isNaN(port) || port <= 0) {
+        return null;
+      }
+      if (debug) {
+        logger.log(chalk.gray(`[DEBUG] Detected port ${port} from docker ps`));
       }
+      return port;
     } catch (error) {
       if (debug) {
         logger.log(chalk.gray(`[DEBUG] Fallback port detection failed: ${error.message}`));

package/lib/utils/infra-status.js

@@ -0,0 +1,123 @@
+/**
+ * AI Fabrix Builder - Infrastructure Status Helpers
+ *
+ * Status-related helper functions for infrastructure management.
+ * Extracted from infra.js to reduce file size.
+ *
+ * @fileoverview Status helper functions for infrastructure management
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const { exec } = require('child_process');
+const { promisify } = require('util');
+const config = require('../config');
+const devConfig = require('./dev-config');
+const containerUtils = require('./infra-containers');
+
+const execAsync = promisify(exec);
+
+/**
+ * Gets the status of infrastructure services
+ * Returns detailed information about running containers
+ *
+ * @async
+ * @function getInfraStatus
+ * @returns {Promise<Object>} Status information for each service
+ *
+ * @example
+ * const status = await getInfraStatus();
+ * // Returns: { postgres: { status: 'running', port: 5432, url: 'localhost:5432' }, ... }
+ */
+async function getInfraStatus() {
+  const devId = await config.getDeveloperId();
+  // Convert string developer ID to number for getDevPorts
+  const devIdNum = parseInt(devId, 10);
+  const ports = devConfig.getDevPorts(devIdNum);
+  const services = {
+    postgres: { port: ports.postgres, url: `localhost:${ports.postgres}` },
+    redis: { port: ports.redis, url: `localhost:${ports.redis}` },
+    pgadmin: { port: ports.pgadmin, url: `http://localhost:${ports.pgadmin}` },
+    'redis-commander': { port: ports.redisCommander, url: `http://localhost:${ports.redisCommander}` }
+  };
+
+  const status = {};
+
+  for (const [serviceName, serviceConfig] of Object.entries(services)) {
+    try {
+      const containerName = await containerUtils.findContainer(serviceName, devId);
+      if (containerName) {
+        const { stdout } = await execAsync(`docker inspect --format='{{.State.Status}}' ${containerName}`);
+        // Normalize status value (trim whitespace and remove quotes)
+        const normalizedStatus = stdout.trim().replace(/['"]/g, '');
+        status[serviceName] = {
+          status: normalizedStatus,
+          port: serviceConfig.port,
+          url: serviceConfig.url
+        };
+      } else {
+        status[serviceName] = {
+          status: 'not running',
+          port: serviceConfig.port,
+          url: serviceConfig.url
+        };
+      }
+    } catch (error) {
+      status[serviceName] = {
+        status: 'not running',
+        port: serviceConfig.port,
+        url: serviceConfig.url
+      };
+    }
+  }
+
+  return status;
+}
+
+/**
+ * Gets status of running application containers
+ * Finds all containers matching pattern aifabrix-dev{id}-* (excluding infrastructure)
+ *
+ * @async
+ * @function getAppStatus
+ * @returns {Promise<Array>} Array of application status objects
+ *
+ * @example
+ * const apps = await getAppStatus();
+ * // Returns: [{ name: 'myapp', container: 'aifabrix-dev1-myapp', port: '3100:3000', status: 'running', url: 'http://localhost:3100' }]
+ */
+async function getAppStatus() {
+  const devId = await config.getDeveloperId();
+  const apps = [];
+
+  try {
+    const filterPattern = devId === 0 ? 'aifabrix-' : `aifabrix-dev${devId}-`;
+    const { stdout } = await execAsync(`docker ps --filter "name=${filterPattern}" --format "{{.Names}}\t{{.Ports}}\t{{.Status}}"`);
+    const lines = stdout.trim().split('\n').filter(line => line.trim() !== '');
+    const infraContainers = devId === 0
+      ? ['aifabrix-postgres', 'aifabrix-redis', 'aifabrix-pgadmin', 'aifabrix-redis-commander']
+      : [`aifabrix-dev${devId}-postgres`, `aifabrix-dev${devId}-redis`, `aifabrix-dev${devId}-pgadmin`, `aifabrix-dev${devId}-redis-commander`];
+    for (const line of lines) {
+      const [containerName, ports, status] = line.split('\t');
+      if (infraContainers.includes(containerName)) continue;
+      const pattern = devId === 0 ? /^aifabrix-(.+)$/ : new RegExp(`^aifabrix-dev${devId}-(.+)$`);
+      const appNameMatch = containerName.match(pattern);
+      if (!appNameMatch) continue;
+      const appName = appNameMatch[1];
+      const portMatch = ports.match(/:(\d+)->\d+\//);
+      const hostPort = portMatch ? portMatch[1] : 'unknown';
+      const url = hostPort !== 'unknown' ? `http://localhost:${hostPort}` : 'unknown';
+      apps.push({ name: appName, container: containerName, port: ports, status: status.trim(), url: url });
+    }
+  } catch (error) {
+    return [];
+  }
+
+  return apps;
+}
+
+module.exports = {
+  getInfraStatus,
+  getAppStatus
+};
+

package/lib/utils/local-secrets.js

@@ -11,11 +11,12 @@
 const fs = require('fs');
 const path = require('path');
 const yaml = require('js-yaml');
-const os = require('os');
 const logger = require('../utils/logger');
+const pathsUtil = require('./paths');
 
 /**
  * Saves a secret to ~/.aifabrix/secrets.local.yaml
+ * Uses paths.getAifabrixHome() to respect config.yaml aifabrix-home override
  * Merges with existing secrets without overwriting other keys
  *
  * @async
@@ -37,7 +38,7 @@ async function saveLocalSecret(key, value) {
     throw new Error('Secret value is required');
   }
 
-  const secretsPath = path.join(os.homedir(), '.aifabrix', 'secrets.local.yaml');
+  const secretsPath = path.join(pathsUtil.getAifabrixHome(), 'secrets.local.yaml');
   const secretsDir = path.dirname(secretsPath);
 
   // Create directory if needed