@aifabrix/builder 2.39.3 → 2.40.2

package/lib/utils/config-format.js

@@ -0,0 +1,154 @@
+/**
+ * Config Format Converter Layer
+ *
+ * Single place for YAML/JSON config I/O. All config loaders and writers use this
+ * layer; internal code works with plain JS objects and JSON Schema only.
+ *
+ * @fileoverview Config format conversion (YAML/JSON) at I/O boundary
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+const YAML_EXTENSIONS = ['.yaml', '.yml'];
+const JSON_EXTENSIONS = ['.json'];
+
+const DEFAULT_YAML_OPTIONS = { indent: 2, lineWidth: 120, noRefs: true };
+
+/**
+ * Parses YAML string to plain JS object (same shape as JSON).
+ * Used when reading .yaml / .yml files.
+ *
+ * @param {string} content - YAML string content
+ * @returns {Object} Plain JS object
+ * @throws {Error} If YAML syntax is invalid
+ */
+function yamlToJson(content) {
+  if (typeof content !== 'string') {
+    throw new Error('yamlToJson expects a string');
+  }
+  try {
+    const parsed = yaml.load(content);
+    return parsed === undefined || parsed === null ? {} : parsed;
+  } catch (error) {
+    throw new Error(`Invalid YAML syntax: ${error.message}`);
+  }
+}
+
+/**
+ * Serializes JS object to YAML string.
+ * Used when writing human-editable config as YAML.
+ *
+ * @param {Object} object - Plain JS object (config)
+ * @param {Object} [options] - js-yaml dump options
+ * @returns {string} YAML string
+ */
+function jsonToYaml(object, options = {}) {
+  if (object === undefined || object === null) {
+    return '';
+  }
+  const opts = { ...DEFAULT_YAML_OPTIONS, ...options };
+  return yaml.dump(object, opts);
+}
+
+/**
+ * Returns whether the file path is treated as YAML by extension.
+ *
+ * @param {string} filePath - File path
+ * @returns {boolean} True if .yaml or .yml
+ */
+function isYamlPath(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  return YAML_EXTENSIONS.includes(ext);
+}
+
+/**
+ * Returns whether the file path is treated as JSON by extension.
+ *
+ * @param {string} filePath - File path
+ * @returns {boolean} True if .json
+ */
+function isJsonPath(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  return JSON_EXTENSIONS.includes(ext);
+}
+
+/**
+ * Reads config file at path; by extension uses yamlToJson or JSON.parse.
+ * Single entry point for "read config file regardless of format".
+ *
+ * @param {string} filePath - Absolute path to config file
+ * @returns {Object} Parsed config object
+ * @throws {Error} If file not found, unreadable, or invalid format
+ */
+function loadConfigFile(filePath) {
+  if (!filePath || typeof filePath !== 'string') {
+    throw new Error('loadConfigFile requires a non-empty file path');
+  }
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`Config file not found: ${filePath}`);
+  }
+  const content = fs.readFileSync(filePath, 'utf8');
+  const ext = path.extname(filePath).toLowerCase();
+  if (YAML_EXTENSIONS.includes(ext)) {
+    return yamlToJson(content);
+  }
+  if (JSON_EXTENSIONS.includes(ext)) {
+    try {
+      const parsed = JSON.parse(content);
+      return parsed === undefined || parsed === null ? {} : parsed;
+    } catch (error) {
+      throw new Error(`Invalid JSON syntax in ${path.basename(filePath)}: ${error.message}`);
+    }
+  }
+  throw new Error(`Unsupported config file extension: ${ext}. Use .yaml, .yml, or .json`);
+}
+
+/**
+ * Writes config object to path as YAML or JSON based on format or path extension.
+ *
+ * @param {string} filePath - Absolute path to write (extension determines format if format omitted)
+ * @param {Object} object - Config object to write
+ * @param {string} [format] - 'yaml' or 'json'; if omitted, inferred from filePath extension
+ * @throws {Error} If format is invalid or write fails
+ */
+function writeConfigFile(filePath, object, format) {
+  if (!filePath || typeof filePath !== 'string') {
+    throw new Error('writeConfigFile requires a non-empty file path');
+  }
+  let targetFormat = format;
+  if (!targetFormat) {
+    const ext = path.extname(filePath).toLowerCase();
+    if (YAML_EXTENSIONS.includes(ext)) {
+      targetFormat = 'yaml';
+    } else if (JSON_EXTENSIONS.includes(ext)) {
+      targetFormat = 'json';
+    } else {
+      throw new Error(`Cannot infer format from path ${filePath}. Use .yaml, .yml, or .json, or pass format.`);
+    }
+  }
+  const normalized = targetFormat.toLowerCase();
+  let content;
+  if (normalized === 'yaml' || normalized === 'yml') {
+    content = jsonToYaml(object);
+  } else if (normalized === 'json') {
+    content = JSON.stringify(object, null, 2);
+  } else {
+    throw new Error(`Invalid format: ${format}. Use 'yaml' or 'json'.`);
+  }
+  fs.writeFileSync(filePath, content, 'utf8');
+}
+
+module.exports = {
+  yamlToJson,
+  jsonToYaml,
+  loadConfigFile,
+  writeConfigFile,
+  isYamlPath,
+  isJsonPath
+};

package/lib/utils/config-paths.js

@@ -41,77 +41,31 @@ async function setPathConfig(getConfigFn, saveConfigFn, key, value, errorMsg) {
   await saveConfigFn(config);
 }
 
-/**
- * Create path configuration functions with config access
- * @param {Function} getConfigFn - Function to get config
- * @param {Function} saveConfigFn - Function to save config
- * @returns {Object} Path configuration functions
- */
-function createPathConfigFunctions(getConfigFn, saveConfigFn) {
+function createHomeAndSecretsPathFunctions(getConfigFn, saveConfigFn) {
   return {
-    /**
-     * Get aifabrix-home override path
-     * @async
-     * @returns {Promise<string|null>} Home path or null
-     */
     async getAifabrixHomeOverride() {
       return getPathConfig(getConfigFn, 'aifabrix-home');
     },
-
-    /**
-     * Set aifabrix-home override path
-     * @async
-     * @param {string} homePath - Home path
-     * @returns {Promise<void>}
-     */
     async setAifabrixHomeOverride(homePath) {
       await setPathConfig(getConfigFn, saveConfigFn, 'aifabrix-home', homePath, 'Home path is required and must be a string');
     },
-
-    /**
-     * Get aifabrix-secrets path
-     * @async
-     * @returns {Promise<string|null>} Secrets path or null
-     */
     async getAifabrixSecretsPath() {
       return getPathConfig(getConfigFn, 'aifabrix-secrets');
     },
-
-    /**
-     * Set aifabrix-secrets path
-     * @async
-     * @param {string} secretsPath - Secrets path
-     * @returns {Promise<void>}
-     */
     async setAifabrixSecretsPath(secretsPath) {
       await setPathConfig(getConfigFn, saveConfigFn, 'aifabrix-secrets', secretsPath, 'Secrets path is required and must be a string');
-    },
+    }
+  };
+}
 
-    /**
-     * Get aifabrix-env-config path
-     * @async
-     * @returns {Promise<string|null>} Env config path or null
-     */
+function createEnvConfigPathFunctions(getConfigFn, saveConfigFn) {
+  return {
     async getAifabrixEnvConfigPath() {
       return getPathConfig(getConfigFn, 'aifabrix-env-config');
     },
-
-    /**
-     * Set aifabrix-env-config path
-     * @async
-     * @param {string} envConfigPath - Env config path
-     * @returns {Promise<void>}
-     */
     async setAifabrixEnvConfigPath(envConfigPath) {
       await setPathConfig(getConfigFn, saveConfigFn, 'aifabrix-env-config', envConfigPath, 'Env config path is required and must be a string');
     },
-
-    /**
-     * Get builder root directory (dirname of aifabrix-env-config when set).
-     * When set, app dirs and generated .env use this instead of cwd/builder.
-     * @async
-     * @returns {Promise<string|null>} Builder root path or null to use cwd/builder
-     */
     async getAifabrixBuilderDir() {
       const envConfigPath = await getPathConfig(getConfigFn, 'aifabrix-env-config');
       return envConfigPath && typeof envConfigPath === 'string' ? path.dirname(envConfigPath) : null;
@@ -119,6 +73,19 @@ function createPathConfigFunctions(getConfigFn, saveConfigFn) {
   };
 }
 
+/**
+ * Create path configuration functions with config access
+ * @param {Function} getConfigFn - Function to get config
+ * @param {Function} saveConfigFn - Function to save config
+ * @returns {Object} Path configuration functions
+ */
+function createPathConfigFunctions(getConfigFn, saveConfigFn) {
+  return {
+    ...createHomeAndSecretsPathFunctions(getConfigFn, saveConfigFn),
+    ...createEnvConfigPathFunctions(getConfigFn, saveConfigFn)
+  };
+}
+
 module.exports = {
   getPathConfig,
   setPathConfig,

package/lib/utils/config-tokens.js

@@ -39,6 +39,7 @@ function normalizeControllerUrl(url) {
  * @param {Function} params.isTokenEncryptedFn - Function to check if token is encrypted
  * @returns {Object} Token management functions
  */
+/* eslint-disable max-lines-per-function -- factory returns many bound helpers; splitting would break encapsulation */
 function createTokenManagementFunctions({
   getConfigFn,
   saveConfigFn,

package/lib/utils/docker-build.js

@@ -63,6 +63,75 @@ function parseDockerBuildProgress(line) {
   return null;
 }
 
+function updateSpinnerFromOutput(output, spinner, lastProgressUpdateRef) {
+  const lines = output.split('\n');
+  for (const line of lines) {
+    const progress = parseDockerBuildProgress(line.trim());
+    if (progress) {
+      const now = Date.now();
+      if (now - lastProgressUpdateRef.current > 200) {
+        spinner.text = `Building image... ${progress}`;
+        lastProgressUpdateRef.current = now;
+      }
+    }
+  }
+}
+
+function handleDockerClose(code, ctx) {
+  const { imageName, tag, stderrBuffer, stdoutBuffer, spinner, resolve, reject } = ctx;
+  if (code === 0) {
+    spinner.succeed(`Image built: ${imageName}:${tag}`);
+    resolve();
+  } else {
+    spinner.fail('Build failed');
+    const errorMessage = stderrBuffer || stdoutBuffer || 'Docker build failed';
+    if (isDockerNotAvailableError(errorMessage)) {
+      reject(new Error('Docker is not running or not installed. Please start Docker Desktop and try again.'));
+    } else {
+      const errorLines = errorMessage.split('\n').filter(line => line.trim());
+      reject(new Error(`Docker build failed: ${errorLines.slice(-5).join('\n')}`));
+    }
+  }
+}
+
+function runDockerBuildProcess(buildOpts) {
+  const { imageName, tag, dockerfilePath, contextPath, spinner, resolve, reject } = buildOpts;
+  const dockerProcess = spawn('docker', ['build', '-t', `${imageName}:${tag}`, '-f', dockerfilePath, contextPath], {
+    shell: process.platform === 'win32'
+  });
+  let stdoutBuffer = '';
+  let stderrBuffer = '';
+  const lastProgressUpdateRef = { current: Date.now() };
+
+  dockerProcess.stdout.on('data', (data) => {
+    const output = data.toString();
+    stdoutBuffer += output;
+    updateSpinnerFromOutput(output, spinner, lastProgressUpdateRef);
+  });
+
+  dockerProcess.stderr.on('data', (data) => {
+    const output = data.toString();
+    stderrBuffer += output;
+    if (!output.toLowerCase().includes('warning')) {
+      updateSpinnerFromOutput(output, spinner, lastProgressUpdateRef);
+    }
+  });
+
+  dockerProcess.on('close', (code) => {
+    handleDockerClose(code, { imageName, tag, stderrBuffer, stdoutBuffer, spinner, resolve, reject });
+  });
+
+  dockerProcess.on('error', (error) => {
+    spinner.fail('Build failed');
+    const msg = error.message || String(error);
+    if (isDockerNotAvailableError(msg)) {
+      reject(new Error('Docker is not running or not installed. Please start Docker Desktop and try again.'));
+    } else {
+      reject(new Error(`Docker build failed: ${msg}`));
+    }
+  });
+}
+
 /**
  * Executes Docker build command with progress indicator
  * @param {string} imageName - Image name to build
@@ -73,29 +142,20 @@ function parseDockerBuildProgress(line) {
  * @throws {Error} If build fails
  */
 async function executeDockerBuild(imageName, dockerfilePath, contextPath, tag) {
-  const spinner = ora({
-    text: 'Starting Docker build...',
-    spinner: 'dots'
-  }).start();
-
-  // Ensure paths are absolute and normalized
+  const spinner = ora({ text: 'Starting Docker build...', spinner: 'dots' }).start();
   const fsSync = require('fs');
   const path = require('path');
-
   dockerfilePath = path.resolve(dockerfilePath);
   contextPath = path.resolve(contextPath);
 
-  // Validate paths exist (skip in test environments)
   const isTestEnv = process.env.NODE_ENV === 'test' ||
                     process.env.JEST_WORKER_ID !== undefined ||
                     typeof jest !== 'undefined';
-
   if (!isTestEnv) {
     if (!fsSync.existsSync(dockerfilePath)) {
       spinner.fail('Build failed');
       throw new Error(`Dockerfile not found: ${dockerfilePath}`);
     }
-
     if (!fsSync.existsSync(contextPath)) {
       spinner.fail('Build failed');
       throw new Error(`Build context path does not exist: ${contextPath}`);
@@ -103,90 +163,7 @@ async function executeDockerBuild(imageName, dockerfilePath, contextPath, tag) {
   }
 
   return new Promise((resolve, reject) => {
-    // Use spawn for streaming output
-    const dockerProcess = spawn('docker', [
-      'build',
-      '-t', `${imageName}:${tag}`,
-      '-f', dockerfilePath,
-      contextPath
-    ], {
-      shell: process.platform === 'win32'
-    });
-
-    let stdoutBuffer = '';
-    let stderrBuffer = '';
-    let lastProgressUpdate = Date.now();
-
-    dockerProcess.stdout.on('data', (data) => {
-      const output = data.toString();
-      stdoutBuffer += output;
-
-      // Parse progress from output lines
-      const lines = output.split('\n');
-      for (const line of lines) {
-        const progress = parseDockerBuildProgress(line.trim());
-        if (progress) {
-          // Update spinner text with progress (throttle updates)
-          const now = Date.now();
-          if (now - lastProgressUpdate > 200) {
-            spinner.text = `Building image... ${progress}`;
-            lastProgressUpdate = now;
-          }
-        }
-      }
-    });
-
-    dockerProcess.stderr.on('data', (data) => {
-      const output = data.toString();
-      stderrBuffer += output;
-
-      // Check for warnings vs errors
-      if (!output.toLowerCase().includes('warning')) {
-        // Parse progress from stderr too (Docker outputs progress to stderr)
-        const lines = output.split('\n');
-        for (const line of lines) {
-          const progress = parseDockerBuildProgress(line.trim());
-          if (progress) {
-            const now = Date.now();
-            if (now - lastProgressUpdate > 200) {
-              spinner.text = `Building image... ${progress}`;
-              lastProgressUpdate = now;
-            }
-          }
-        }
-      }
-    });
-
-    dockerProcess.on('close', (code) => {
-      if (code === 0) {
-        spinner.succeed(`Image built: ${imageName}:${tag}`);
-        resolve();
-      } else {
-        spinner.fail('Build failed');
-
-        const errorMessage = stderrBuffer || stdoutBuffer || 'Docker build failed';
-
-        if (isDockerNotAvailableError(errorMessage)) {
-          reject(new Error('Docker is not running or not installed. Please start Docker Desktop and try again.'));
-        } else {
-          // Show last few lines of error output
-          const errorLines = errorMessage.split('\n').filter(line => line.trim());
-          const lastError = errorLines.slice(-5).join('\n');
-          reject(new Error(`Docker build failed: ${lastError}`));
-        }
-      }
-    });
-
-    dockerProcess.on('error', (error) => {
-      spinner.fail('Build failed');
-      const errorMessage = error.message || String(error);
-
-      if (isDockerNotAvailableError(errorMessage)) {
-        reject(new Error('Docker is not running or not installed. Please start Docker Desktop and try again.'));
-      } else {
-        reject(new Error(`Docker build failed: ${errorMessage}`));
-      }
-    });
+    runDockerBuildProcess({ imageName, tag, dockerfilePath, contextPath, spinner, resolve, reject });
   });
 }
 

package/lib/utils/dockerfile-utils.js

@@ -103,7 +103,7 @@ function checkTemplateDockerfile(builderPath, appName, forceTemplate) {
 }
 
 /**
- * Checks for custom Dockerfile from variables.yaml
+ * Checks for custom Dockerfile from application config
  * @function checkProjectDockerfile
  * @param {string} builderPath - Builder directory path
  * @param {string} appName - Application name

package/lib/utils/env-copy.js

@@ -50,8 +50,8 @@ function readDeveloperIdFromConfig(config) {
 
 /**
  * Resolve output path for env file
- * @param {string} rawOutputPath - Raw output path from variables.yaml
- * @param {string} variablesPath - Path to variables.yaml
+ * @param {string} rawOutputPath - Raw output path from application config
+ * @param {string} variablesPath - Path to application config
  * @returns {string} Resolved output path
  */
 function resolveEnvOutputPath(rawOutputPath, variablesPath) {
@@ -154,7 +154,7 @@ function extractEnvVarsFromContent(envContent, envVars) {
  * Patch env content for local development
  * @async
  * @param {string} envContent - Original env content
- * @param {Object} variables - Variables from variables.yaml
+ * @param {Object} variables - Variables from application config
  * @returns {Promise<string>} Patched env content
  */
 async function patchEnvContentForLocal(envContent, variables) {
@@ -186,7 +186,7 @@ async function patchEnvContentForLocal(envContent, variables) {
  * @async
  * @function processEnvVariables
  * @param {string} envPath - Path to generated .env file
- * @param {string} variablesPath - Path to variables.yaml
+ * @param {string} variablesPath - Path to application config
  * @param {string} appName - Application name (for regenerating with local env)
  * @param {string} [secretsPath] - Path to secrets file (optional, for regenerating)
  */

package/lib/utils/env-ports.js

@@ -1,7 +1,7 @@
 /**
  * Environment port utilities
  *
- * @fileoverview Update container PORT based on variables.yaml and developer-id offset
+ * @fileoverview Update container PORT based on application config and developer-id offset
  * @author AI Fabrix Team
  * @version 2.0.0
  */
@@ -17,7 +17,7 @@ const { getLocalPort } = require('./port-resolver');
  * Update PORT in the container's .env file to use variables.port (+offset)
  * @function updateContainerPortInEnvFile
  * @param {string} envPath - Path to .env
- * @param {string} variablesPath - Path to variables.yaml
+ * @param {string} variablesPath - Path to application config
  */
 /**
  * Gets developer ID from environment variable or config file

package/lib/utils/error-formatter.js

@@ -187,7 +187,7 @@ function formatMissingDbPasswordError(appKey, opts = {}) {
       'Add ' + passwordKey + '=your_secret to your .env file. For multiple databases you need DB_0_PASSWORD, DB_1_PASSWORD, etc.';
   }
   return 'Missing required password variable DB_0_PASSWORD or DB_PASSWORD in .env file for application \'' + appKey + '\'. ' +
-    'This app has requires.database or databases in variables.yaml. Add DB_0_PASSWORD=your_secret or DB_PASSWORD=your_secret to builder/' + appKey + '/.env (or run \'aifabrix resolve ' + appKey + '\'), or set requires.database: false in variables.yaml if not needed.';
+    'This app has requires.database or databases in application.yaml. Add DB_0_PASSWORD=your_secret or DB_PASSWORD=your_secret to builder/' + appKey + '/.env (or run \'aifabrix resolve ' + appKey + '\'), or set requires.database: false in application.yaml if not needed.';
 }
 
 module.exports = {

package/lib/utils/error-formatters/validation-errors.js

@@ -97,7 +97,7 @@ function addValidationGuidance(lines, hasErrors) {
     return;
   }
   lines.push(chalk.gray('Tips:'));
-  lines.push(chalk.gray('  • Check your variables.yaml file for the correct field values'));
+  lines.push(chalk.gray('  • Check your application.yaml file for the correct field values'));
   lines.push(chalk.gray('  • Verify field names match the expected schema'));
   lines.push(chalk.gray('  • Ensure required fields are present and valid'));
   lines.push('');

package/lib/utils/external-readme.js

@@ -51,11 +51,18 @@ function normalizeDatasources(datasources, systemKey) {
     let fileName = datasource.fileName || datasource.file;
     if (!fileName) {
       const key = datasource.key || '';
-      // Extract entity from keys like "hubspot-deploy-company" -> "company"
-      const entity = (systemKey && key.startsWith(`${systemKey}-deploy-`))
-        ? key.slice(`${systemKey}-deploy-`.length)
-        : entityType;
-      fileName = systemKey ? `${systemKey}-datasource-${entity}.json` : `${entity}.json`;
+      // Suffix matches split getExternalDatasourceFileName for consistent README and file names
+      let suffix;
+      if (key.startsWith(`${systemKey}-deploy-`)) {
+        suffix = key.slice(`${systemKey}-deploy-`.length);
+      } else if (systemKey && key.startsWith(`${systemKey}-`)) {
+        suffix = key.slice(systemKey.length + 1);
+      } else if (key) {
+        suffix = key;
+      } else {
+        suffix = entityType;
+      }
+      fileName = systemKey ? `${systemKey}-datasource-${suffix}.yaml` : `${suffix}.yaml`;
     }
     return { entityType, displayName, fileName };
   });

package/lib/utils/external-system-test-helpers.js

@@ -12,6 +12,7 @@
 const fs = require('fs').promises;
 const path = require('path');
 const { testDatasourceViaPipeline } = require('../api/pipeline.api');
+const { requireBearerForDataplanePipeline } = require('./token-manager');
 
 /**
  * Retry API call with exponential backoff
@@ -51,6 +52,7 @@ async function retryApiCall(fn, maxRetries = 3, backoffMs = 1000) {
  * @returns {Promise<Object>} Test response
  */
 async function callPipelineTestEndpoint({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout = 30000 }) {
+  requireBearerForDataplanePipeline(authConfig);
   const response = await retryApiCall(async() => {
     return await testDatasourceViaPipeline({
       dataplaneUrl,