@aifabrix/builder 2.39.2 → 2.40.0

package/lib/utils/docker-build.js

@@ -63,6 +63,75 @@ function parseDockerBuildProgress(line) {
   return null;
 }
 
+function updateSpinnerFromOutput(output, spinner, lastProgressUpdateRef) {
+  const lines = output.split('\n');
+  for (const line of lines) {
+    const progress = parseDockerBuildProgress(line.trim());
+    if (progress) {
+      const now = Date.now();
+      if (now - lastProgressUpdateRef.current > 200) {
+        spinner.text = `Building image... ${progress}`;
+        lastProgressUpdateRef.current = now;
+      }
+    }
+  }
+}
+
+function handleDockerClose(code, ctx) {
+  const { imageName, tag, stderrBuffer, stdoutBuffer, spinner, resolve, reject } = ctx;
+  if (code === 0) {
+    spinner.succeed(`Image built: ${imageName}:${tag}`);
+    resolve();
+  } else {
+    spinner.fail('Build failed');
+    const errorMessage = stderrBuffer || stdoutBuffer || 'Docker build failed';
+    if (isDockerNotAvailableError(errorMessage)) {
+      reject(new Error('Docker is not running or not installed. Please start Docker Desktop and try again.'));
+    } else {
+      const errorLines = errorMessage.split('\n').filter(line => line.trim());
+      reject(new Error(`Docker build failed: ${errorLines.slice(-5).join('\n')}`));
+    }
+  }
+}
+
+function runDockerBuildProcess(buildOpts) {
+  const { imageName, tag, dockerfilePath, contextPath, spinner, resolve, reject } = buildOpts;
+  const dockerProcess = spawn('docker', ['build', '-t', `${imageName}:${tag}`, '-f', dockerfilePath, contextPath], {
+    shell: process.platform === 'win32'
+  });
+  let stdoutBuffer = '';
+  let stderrBuffer = '';
+  const lastProgressUpdateRef = { current: Date.now() };
+
+  dockerProcess.stdout.on('data', (data) => {
+    const output = data.toString();
+    stdoutBuffer += output;
+    updateSpinnerFromOutput(output, spinner, lastProgressUpdateRef);
+  });
+
+  dockerProcess.stderr.on('data', (data) => {
+    const output = data.toString();
+    stderrBuffer += output;
+    if (!output.toLowerCase().includes('warning')) {
+      updateSpinnerFromOutput(output, spinner, lastProgressUpdateRef);
+    }
+  });
+
+  dockerProcess.on('close', (code) => {
+    handleDockerClose(code, { imageName, tag, stderrBuffer, stdoutBuffer, spinner, resolve, reject });
+  });
+
+  dockerProcess.on('error', (error) => {
+    spinner.fail('Build failed');
+    const msg = error.message || String(error);
+    if (isDockerNotAvailableError(msg)) {
+      reject(new Error('Docker is not running or not installed. Please start Docker Desktop and try again.'));
+    } else {
+      reject(new Error(`Docker build failed: ${msg}`));
+    }
+  });
+}
+
 /**
  * Executes Docker build command with progress indicator
  * @param {string} imageName - Image name to build
@@ -73,29 +142,20 @@ function parseDockerBuildProgress(line) {
  * @throws {Error} If build fails
  */
 async function executeDockerBuild(imageName, dockerfilePath, contextPath, tag) {
-  const spinner = ora({
-    text: 'Starting Docker build...',
-    spinner: 'dots'
-  }).start();
-
-  // Ensure paths are absolute and normalized
+  const spinner = ora({ text: 'Starting Docker build...', spinner: 'dots' }).start();
   const fsSync = require('fs');
   const path = require('path');
-
   dockerfilePath = path.resolve(dockerfilePath);
   contextPath = path.resolve(contextPath);
 
-  // Validate paths exist (skip in test environments)
   const isTestEnv = process.env.NODE_ENV === 'test' ||
                     process.env.JEST_WORKER_ID !== undefined ||
                     typeof jest !== 'undefined';
-
   if (!isTestEnv) {
     if (!fsSync.existsSync(dockerfilePath)) {
       spinner.fail('Build failed');
       throw new Error(`Dockerfile not found: ${dockerfilePath}`);
     }
-
     if (!fsSync.existsSync(contextPath)) {
       spinner.fail('Build failed');
       throw new Error(`Build context path does not exist: ${contextPath}`);
@@ -103,90 +163,7 @@ async function executeDockerBuild(imageName, dockerfilePath, contextPath, tag) {
   }
 
   return new Promise((resolve, reject) => {
-    // Use spawn for streaming output
-    const dockerProcess = spawn('docker', [
-      'build',
-      '-t', `${imageName}:${tag}`,
-      '-f', dockerfilePath,
-      contextPath
-    ], {
-      shell: process.platform === 'win32'
-    });
-
-    let stdoutBuffer = '';
-    let stderrBuffer = '';
-    let lastProgressUpdate = Date.now();
-
-    dockerProcess.stdout.on('data', (data) => {
-      const output = data.toString();
-      stdoutBuffer += output;
-
-      // Parse progress from output lines
-      const lines = output.split('\n');
-      for (const line of lines) {
-        const progress = parseDockerBuildProgress(line.trim());
-        if (progress) {
-          // Update spinner text with progress (throttle updates)
-          const now = Date.now();
-          if (now - lastProgressUpdate > 200) {
-            spinner.text = `Building image... ${progress}`;
-            lastProgressUpdate = now;
-          }
-        }
-      }
-    });
-
-    dockerProcess.stderr.on('data', (data) => {
-      const output = data.toString();
-      stderrBuffer += output;
-
-      // Check for warnings vs errors
-      if (!output.toLowerCase().includes('warning')) {
-        // Parse progress from stderr too (Docker outputs progress to stderr)
-        const lines = output.split('\n');
-        for (const line of lines) {
-          const progress = parseDockerBuildProgress(line.trim());
-          if (progress) {
-            const now = Date.now();
-            if (now - lastProgressUpdate > 200) {
-              spinner.text = `Building image... ${progress}`;
-              lastProgressUpdate = now;
-            }
-          }
-        }
-      }
-    });
-
-    dockerProcess.on('close', (code) => {
-      if (code === 0) {
-        spinner.succeed(`Image built: ${imageName}:${tag}`);
-        resolve();
-      } else {
-        spinner.fail('Build failed');
-
-        const errorMessage = stderrBuffer || stdoutBuffer || 'Docker build failed';
-
-        if (isDockerNotAvailableError(errorMessage)) {
-          reject(new Error('Docker is not running or not installed. Please start Docker Desktop and try again.'));
-        } else {
-          // Show last few lines of error output
-          const errorLines = errorMessage.split('\n').filter(line => line.trim());
-          const lastError = errorLines.slice(-5).join('\n');
-          reject(new Error(`Docker build failed: ${lastError}`));
-        }
-      }
-    });
-
-    dockerProcess.on('error', (error) => {
-      spinner.fail('Build failed');
-      const errorMessage = error.message || String(error);
-
-      if (isDockerNotAvailableError(errorMessage)) {
-        reject(new Error('Docker is not running or not installed. Please start Docker Desktop and try again.'));
-      } else {
-        reject(new Error(`Docker build failed: ${errorMessage}`));
-      }
-    });
+    runDockerBuildProcess({ imageName, tag, dockerfilePath, contextPath, spinner, resolve, reject });
   });
 }
 

package/lib/utils/dockerfile-utils.js

@@ -103,7 +103,7 @@ function checkTemplateDockerfile(builderPath, appName, forceTemplate) {
 }
 
 /**
- * Checks for custom Dockerfile from variables.yaml
+ * Checks for custom Dockerfile from application config
  * @function checkProjectDockerfile
  * @param {string} builderPath - Builder directory path
  * @param {string} appName - Application name

package/lib/utils/env-copy.js

@@ -50,8 +50,8 @@ function readDeveloperIdFromConfig(config) {
 
 /**
  * Resolve output path for env file
- * @param {string} rawOutputPath - Raw output path from variables.yaml
- * @param {string} variablesPath - Path to variables.yaml
+ * @param {string} rawOutputPath - Raw output path from application config
+ * @param {string} variablesPath - Path to application config
  * @returns {string} Resolved output path
  */
 function resolveEnvOutputPath(rawOutputPath, variablesPath) {
@@ -154,7 +154,7 @@ function extractEnvVarsFromContent(envContent, envVars) {
  * Patch env content for local development
  * @async
  * @param {string} envContent - Original env content
- * @param {Object} variables - Variables from variables.yaml
+ * @param {Object} variables - Variables from application config
  * @returns {Promise<string>} Patched env content
  */
 async function patchEnvContentForLocal(envContent, variables) {
@@ -186,7 +186,7 @@ async function patchEnvContentForLocal(envContent, variables) {
  * @async
  * @function processEnvVariables
  * @param {string} envPath - Path to generated .env file
- * @param {string} variablesPath - Path to variables.yaml
+ * @param {string} variablesPath - Path to application config
  * @param {string} appName - Application name (for regenerating with local env)
  * @param {string} [secretsPath] - Path to secrets file (optional, for regenerating)
  */

package/lib/utils/env-ports.js

@@ -1,7 +1,7 @@
 /**
  * Environment port utilities
  *
- * @fileoverview Update container PORT based on variables.yaml and developer-id offset
+ * @fileoverview Update container PORT based on application config and developer-id offset
  * @author AI Fabrix Team
  * @version 2.0.0
  */
@@ -17,7 +17,7 @@ const { getLocalPort } = require('./port-resolver');
  * Update PORT in the container's .env file to use variables.port (+offset)
  * @function updateContainerPortInEnvFile
  * @param {string} envPath - Path to .env
- * @param {string} variablesPath - Path to variables.yaml
+ * @param {string} variablesPath - Path to application config
  */
 /**
  * Gets developer ID from environment variable or config file

package/lib/utils/error-formatter.js

@@ -187,7 +187,7 @@ function formatMissingDbPasswordError(appKey, opts = {}) {
       'Add ' + passwordKey + '=your_secret to your .env file. For multiple databases you need DB_0_PASSWORD, DB_1_PASSWORD, etc.';
   }
   return 'Missing required password variable DB_0_PASSWORD or DB_PASSWORD in .env file for application \'' + appKey + '\'. ' +
-    'This app has requires.database or databases in variables.yaml. Add DB_0_PASSWORD=your_secret or DB_PASSWORD=your_secret to builder/' + appKey + '/.env (or run \'aifabrix resolve ' + appKey + '\'), or set requires.database: false in variables.yaml if not needed.';
+    'This app has requires.database or databases in application.yaml. Add DB_0_PASSWORD=your_secret or DB_PASSWORD=your_secret to builder/' + appKey + '/.env (or run \'aifabrix resolve ' + appKey + '\'), or set requires.database: false in application.yaml if not needed.';
 }
 
 module.exports = {

package/lib/utils/error-formatters/validation-errors.js

@@ -97,7 +97,7 @@ function addValidationGuidance(lines, hasErrors) {
     return;
   }
   lines.push(chalk.gray('Tips:'));
-  lines.push(chalk.gray('  • Check your variables.yaml file for the correct field values'));
+  lines.push(chalk.gray('  • Check your application.yaml file for the correct field values'));
   lines.push(chalk.gray('  • Verify field names match the expected schema'));
   lines.push(chalk.gray('  • Ensure required fields are present and valid'));
   lines.push('');

package/lib/utils/external-readme.js

@@ -51,11 +51,18 @@ function normalizeDatasources(datasources, systemKey) {
     let fileName = datasource.fileName || datasource.file;
     if (!fileName) {
       const key = datasource.key || '';
-      // Extract entity from keys like "hubspot-deploy-company" -> "company"
-      const entity = (systemKey && key.startsWith(`${systemKey}-deploy-`))
-        ? key.slice(`${systemKey}-deploy-`.length)
-        : entityType;
-      fileName = systemKey ? `${systemKey}-datasource-${entity}.json` : `${entity}.json`;
+      // Suffix matches split getExternalDatasourceFileName for consistent README and file names
+      let suffix;
+      if (key.startsWith(`${systemKey}-deploy-`)) {
+        suffix = key.slice(`${systemKey}-deploy-`.length);
+      } else if (systemKey && key.startsWith(`${systemKey}-`)) {
+        suffix = key.slice(systemKey.length + 1);
+      } else if (key) {
+        suffix = key;
+      } else {
+        suffix = entityType;
+      }
+      fileName = systemKey ? `${systemKey}-datasource-${suffix}.yaml` : `${suffix}.yaml`;
     }
     return { entityType, displayName, fileName };
   });

package/lib/utils/external-system-test-helpers.js

@@ -12,6 +12,7 @@
 const fs = require('fs').promises;
 const path = require('path');
 const { testDatasourceViaPipeline } = require('../api/pipeline.api');
+const { requireBearerForDataplanePipeline } = require('./token-manager');
 
 /**
  * Retry API call with exponential backoff
@@ -51,6 +52,7 @@ async function retryApiCall(fn, maxRetries = 3, backoffMs = 1000) {
  * @returns {Promise<Object>} Test response
  */
 async function callPipelineTestEndpoint({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout = 30000 }) {
+  requireBearerForDataplanePipeline(authConfig);
   const response = await retryApiCall(async() => {
     return await testDatasourceViaPipeline({
       dataplaneUrl,

package/lib/utils/health-check.js

@@ -231,6 +231,60 @@ function parseHealthResponse(data, statusCode) {
   }
 }
 
+function handleHealthResponse(res, data, debug, resolve) {
+  const isHealthy = parseHealthResponse(data, res.statusCode);
+  if (debug) {
+    const truncatedData = data.length > 200 ? data.substring(0, 200) + '...' : data;
+    logger.log(chalk.gray(`[DEBUG] Response body: ${truncatedData}`));
+    logger.log(chalk.gray(`[DEBUG] Health check result: ${isHealthy ? 'healthy' : 'unhealthy'}`));
+  }
+  resolve(isHealthy);
+}
+
+function doHealthCheckRequest(healthCheckUrl, debug, resolve, reject) {
+  try {
+    const urlObj = new URL(healthCheckUrl);
+    const options = {
+      hostname: urlObj.hostname,
+      port: urlObj.port || (urlObj.protocol === 'https:' ? 443 : 80),
+      path: urlObj.pathname + urlObj.search,
+      method: 'GET'
+    };
+    if (debug) {
+      logger.log(chalk.gray(`[DEBUG] Health check request: ${healthCheckUrl}`));
+      logger.log(chalk.gray(`[DEBUG] Request options: ${JSON.stringify(options, null, 2)}`));
+    }
+    // eslint-disable-next-line prefer-const
+    let timeoutId;
+    const req = http.request(options, (res) => {
+      clearTimeout(timeoutId);
+      let data = '';
+      if (debug) {
+        logger.log(chalk.gray(`[DEBUG] Response status code: ${res.statusCode}`));
+        logger.log(chalk.gray(`[DEBUG] Response headers: ${JSON.stringify(res.headers, null, 2)}`));
+      }
+      res.on('data', (chunk) => {
+        data += chunk;
+      });
+      res.on('end', () => handleHealthResponse(res, data, debug, resolve));
+    });
+    timeoutId = setTimeout(() => {
+      if (debug) logger.log(chalk.gray('[DEBUG] Health check request timeout after 5 seconds'));
+      req.destroy();
+      resolve(false);
+    }, 5000);
+    req.on('error', (error) => {
+      clearTimeout(timeoutId);
+      if (debug) logger.log(chalk.gray(`[DEBUG] Health check request error: ${error.message}`));
+      resolve(false);
+    });
+    req.end();
+  } catch (error) {
+    if (debug) logger.log(chalk.gray(`[DEBUG] Health check exception: ${error.message}`));
+    reject(error);
+  }
+}
+
 /**
  * Checks health endpoint
  * @async
@@ -242,72 +296,7 @@ function parseHealthResponse(data, statusCode) {
  */
 async function checkHealthEndpoint(healthCheckUrl, debug = false) {
   return new Promise((resolve, reject) => {
-    try {
-      const urlObj = new URL(healthCheckUrl);
-      const options = {
-        hostname: urlObj.hostname,
-        port: urlObj.port || (urlObj.protocol === 'https:' ? 443 : 80),
-        path: urlObj.pathname + urlObj.search,
-        method: 'GET'
-      };
-
-      if (debug) {
-        logger.log(chalk.gray(`[DEBUG] Health check request: ${healthCheckUrl}`));
-        logger.log(chalk.gray(`[DEBUG] Request options: ${JSON.stringify(options, null, 2)}`));
-      }
-
-      // Declare timeoutId before creating req so it can be used in callbacks
-      // eslint-disable-next-line prefer-const
-      let timeoutId;
-
-      const req = http.request(options, (res) => {
-        clearTimeout(timeoutId);
-        let data = '';
-        if (debug) {
-          logger.log(chalk.gray(`[DEBUG] Response status code: ${res.statusCode}`));
-          logger.log(chalk.gray(`[DEBUG] Response headers: ${JSON.stringify(res.headers, null, 2)}`));
-        }
-        res.on('data', (chunk) => {
-          data += chunk;
-        });
-        res.on('end', () => {
-          if (debug) {
-            const truncatedData = data.length > 200 ? data.substring(0, 200) + '...' : data;
-            logger.log(chalk.gray(`[DEBUG] Response body: ${truncatedData}`));
-          }
-          const isHealthy = parseHealthResponse(data, res.statusCode);
-          if (debug) {
-            logger.log(chalk.gray(`[DEBUG] Health check result: ${isHealthy ? 'healthy' : 'unhealthy'}`));
-          }
-          resolve(isHealthy);
-        });
-      });
-
-      // Set timeout for the request using setTimeout
-      timeoutId = setTimeout(() => {
-        if (debug) {
-          logger.log(chalk.gray('[DEBUG] Health check request timeout after 5 seconds'));
-        }
-        req.destroy();
-        resolve(false);
-      }, 5000);
-
-      req.on('error', (error) => {
-        clearTimeout(timeoutId);
-        if (debug) {
-          logger.log(chalk.gray(`[DEBUG] Health check request error: ${error.message}`));
-        }
-        resolve(false);
-      });
-
-      req.end();
-    } catch (error) {
-      if (debug) {
-        logger.log(chalk.gray(`[DEBUG] Health check exception: ${error.message}`));
-      }
-      // Re-throw exceptions (not just network errors)
-      reject(error);
-    }
+    doHealthCheckRequest(healthCheckUrl, debug, resolve, reject);
   });
 }
 

package/lib/utils/image-version.js

@@ -11,11 +11,9 @@
 
 const { exec } = require('child_process');
 const { promisify } = require('util');
-const fs = require('fs').promises;
-const fsSync = require('fs');
-const path = require('path');
-const yaml = require('js-yaml');
 const { getBuilderPath } = require('./paths');
+const { resolveApplicationConfigPath } = require('./app-config-resolver');
+const { loadConfigFile, writeConfigFile } = require('./config-format');
 const composeGenerator = require('./compose-generator');
 const containerHelpers = require('./app-run-containers');
 
@@ -86,7 +84,7 @@ function compareSemver(a, b) {
 
 /**
  * Resolves version for external app (app.version or externalIntegration.version)
- * @param {Object} variables - Parsed variables.yaml
+ * @param {Object} variables - Parsed application config
  * @returns {string}
  */
 function resolveExternalVersion(variables) {
@@ -125,9 +123,9 @@ async function resolveRegularVersion(imageName, imageTag, templateVersion) {
  * Resolves version for an app: from image when image exists and template empty or smaller
  * @async
  * @param {string} appName - Application name
- * @param {Object} variables - Parsed variables.yaml
+ * @param {Object} variables - Parsed application config
  * @param {Object} [options] - Options
- * @param {boolean} [options.updateBuilder] - When true, update builder/variables.yaml if fromImage
+ * @param {boolean} [options.updateBuilder] - When true, update builder application config if fromImage
  * @param {string} [options.builderPath] - Builder path (defaults to getBuilderPath(appName))
  * @returns {Promise<{ version: string, fromImage: boolean, updated: boolean }>}
  */
@@ -164,37 +162,30 @@ async function resolveVersionForApp(appName, variables, options = {}) {
   let updated = false;
   if (fromImage && options.updateBuilder) {
     const builderPath = options.builderPath || getBuilderPath(appName);
-    updated = await updateAppVersionInVariablesYaml(builderPath, version);
+    updated = updateAppVersionInVariablesYaml(builderPath, version);
   }
 
   return { version, fromImage, updated };
 }
 
 /**
- * Updates app.version in builder variables.yaml
- * @async
+ * Updates app.version in builder application config
  * @param {string} builderPath - Path to builder app directory
  * @param {string} version - Version to set
- * @returns {Promise<boolean>} True if file was updated
+ * @returns {boolean} True if file was updated
  */
-async function updateAppVersionInVariablesYaml(builderPath, version) {
+function updateAppVersionInVariablesYaml(builderPath, version) {
   if (!builderPath || !version || typeof version !== 'string') {
     return false;
   }
-  const variablesPath = path.join(builderPath, 'variables.yaml');
-  if (!fsSync.existsSync(variablesPath)) {
-    return false;
-  }
-
   try {
-    const content = await fs.readFile(variablesPath, 'utf8');
-    const parsed = yaml.load(content) || {};
+    const configPath = resolveApplicationConfigPath(builderPath);
+    const parsed = loadConfigFile(configPath) || {};
     if (!parsed.app) {
       parsed.app = {};
     }
     parsed.app.version = version;
-    const dumped = yaml.dump(parsed, { lineWidth: -1 });
-    await fs.writeFile(variablesPath, dumped, { mode: 0o644, encoding: 'utf8' });
+    writeConfigFile(configPath, parsed);
     return true;
   } catch {
     return false;