@aifabrix/builder 2.45.5 → 2.47.0

package/lib/datasource/validate.js

@@ -10,6 +10,7 @@
 
 const path = require('path');
 const fs = require('fs');
+const { loadConfigFile } = require('../utils/config-format');
 const pathsUtil = require('../utils/paths');
 const {
   datasourceIdentifierMatchesFile,
@@ -22,7 +23,16 @@ const { validateFieldReferences } = require('./field-reference-validator');
 const { validateAbac } = require('./abac-validator');
 const { buildDatasourceValidateSummary } = require('./datasource-validate-summary');
 
-const EXCLUDE_JSON_NAMES = new Set(['application.json', 'rbac.json', 'package.json', 'package-lock.json']);
+const DATASOURCE_CONFIG_EXTENSIONS = new Set(['.json', '.yaml', '.yml']);
+const EXCLUDE_CONFIG_NAMES = new Set([
+  'application.json',
+  'application.yaml',
+  'application.yml',
+  'rbac.json',
+  'rbac.yaml',
+  'package.json',
+  'package-lock.json'
+]);
 
 /**
  * Resolve identifier to an existing file path, or null.
@@ -64,11 +74,14 @@ function pickIntegrationAppForDatasourceKey(datasourceKey, appNames) {
  * @param {string} fileName
  * @returns {boolean}
  */
-function shouldScanJsonDatasourceCandidate(fileName) {
+function shouldScanDatasourceConfigCandidate(fileName) {
   const lower = fileName.toLowerCase();
-  if (!lower.endsWith('.json')) return false;
-  if (EXCLUDE_JSON_NAMES.has(fileName)) return false;
+  const ext = path.extname(lower);
+  if (!DATASOURCE_CONFIG_EXTENSIONS.has(ext)) return false;
+  if (EXCLUDE_CONFIG_NAMES.has(fileName)) return false;
   if (lower.endsWith('-deploy.json') || lower.endsWith('-system.json')) return false;
+  if (lower.endsWith('-deploy.yaml') || lower.endsWith('-deploy.yml')) return false;
+  if (lower.endsWith('-system.yaml') || lower.endsWith('-system.yml')) return false;
   return true;
 }
 
@@ -86,13 +99,13 @@ function isDatasourceNamedJson(fileName) {
  * @param {string} datasourceKey
  * @returns {string[]}
  */
-function collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey) {
+function collectConfigPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey) {
   const hits = [];
   for (const name of scanOrder) {
     const full = path.join(dir, name);
     let parsed;
     try {
-      parsed = JSON.parse(fs.readFileSync(full, 'utf8'));
+      parsed = loadConfigFile(full);
     } catch {
       continue;
     }
@@ -115,14 +128,14 @@ function resolvePathFromIntegrationFolderScan(appName, datasourceKey) {
     throw new Error(`Integration folder not found: ${dir}`);
   }
   const entries = fs.readdirSync(dir);
-  const jsonFiles = entries.filter(shouldScanJsonDatasourceCandidate);
-  const prefer = jsonFiles.filter(isDatasourceNamedJson);
-  const scanOrder = [...prefer, ...jsonFiles.filter((f) => !prefer.includes(f))];
-  const hits = collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey);
+  const configFiles = entries.filter(shouldScanDatasourceConfigCandidate);
+  const prefer = configFiles.filter(isDatasourceNamedJson);
+  const scanOrder = [...prefer, ...configFiles.filter((f) => !prefer.includes(f))];
+  const hits = collectConfigPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey);
   if (hits.length === 0) {
     throw new Error(
-      `No datasource JSON matching "${datasourceKey}" under integration/${appName}/. ` +
-        `Checked: ${scanOrder.length ? scanOrder.join(', ') : '(no JSON candidates)'}`
+      `No datasource config matching "${datasourceKey}" under integration/${appName}/. ` +
+        `Checked: ${scanOrder.length ? scanOrder.join(', ') : '(no JSON/YAML candidates)'}`
     );
   }
   if (hits.length > 1) {
@@ -167,7 +180,8 @@ function resolveValidateInputPath(identifier) {
   }
   const asFile = tryResolveAsExistingFile(trimmed);
   if (asFile) return asFile;
-  if (trimmed.toLowerCase().endsWith('.json')) {
+  const ext = path.extname(trimmed).toLowerCase();
+  if (DATASOURCE_CONFIG_EXTENSIONS.has(ext)) {
     throw new Error(`File not found: ${trimmed}`);
   }
   return resolvePathFromIntegrationDatasourceKey(trimmed);
@@ -187,6 +201,18 @@ function resolveValidateInputPath(identifier) {
  * const result = await validateDatasourceFile('./hubspot-deal.json');
  * // Returns: { valid: true, errors: [], warnings: [], resolvedPath: '...' }
  */
+function buildParseFailureResult(filePath, error) {
+  const message =
+    error && error.message ? String(error.message) : 'Invalid config file';
+  return {
+    valid: false,
+    errors: [message],
+    warnings: [],
+    resolvedPath: filePath,
+    summary: null
+  };
+}
+
 async function validateDatasourceFile(filePathOrKey) {
   if (!filePathOrKey || typeof filePathOrKey !== 'string') {
     throw new Error('File path is required and must be a string');
@@ -194,19 +220,11 @@ async function validateDatasourceFile(filePathOrKey) {
 
   const filePath = resolveValidateInputPath(filePathOrKey.trim());
 
-  const content = fs.readFileSync(filePath, 'utf8');
   let parsed;
-
   try {
-    parsed = JSON.parse(content);
+    parsed = loadConfigFile(filePath);
   } catch (error) {
-    return {
-      valid: false,
-      errors: [`Invalid JSON syntax: ${error.message}`],
-      warnings: [],
-      resolvedPath: filePath,
-      summary: null
-    };
+    return buildParseFailureResult(filePath, error);
   }
 
   const summary = buildDatasourceValidateSummary(parsed);

package/lib/deployment/environment-poll-status.js

@@ -0,0 +1,138 @@
+/**
+ * Poll helpers for environment deploy status (deployment-by-ID).
+ *
+ * @fileoverview Parse pipeline/environments deployment poll responses and fail fast on terminal HTTP errors
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+const TERMINAL_POLL_HTTP_STATUSES = new Set([400, 401, 403, 404, 410, 422]);
+
+/**
+ * @param {Object|null|undefined} response - API client response
+ * @returns {boolean}
+ */
+function isTerminalPollApiResponse(response) {
+  return Boolean(
+    response &&
+    response.success === false &&
+    typeof response.status === 'number' &&
+    TERMINAL_POLL_HTTP_STATUSES.has(response.status)
+  );
+}
+
+/**
+ * @param {Object|null|undefined} response - Successful API client response
+ * @returns {Object|null} Deployment record or null
+ */
+function extractDeploymentRecord(response) {
+  if (!response || response.success === false) {
+    return null;
+  }
+  if (response?.data?.data) {
+    return response.data.data;
+  }
+  if (response?.data && typeof response.data === 'object' && response.data.status) {
+    return response.data;
+  }
+  return null;
+}
+
+/**
+ * @param {Object|null|undefined} pipelineRes
+ * @param {Object|null|undefined} envRes
+ * @returns {Error}
+ */
+/**
+ * @param {string} requestedEnvKey
+ * @param {Error} error
+ * @returns {Error}
+ */
+function enrichPollTerminalError(requestedEnvKey, error) {
+  if (!error?.pollTerminal || error.status !== 404) {
+    return error;
+  }
+  const base = error.message || '';
+  if (!base.includes('Environment with key')) {
+    return error;
+  }
+  error.message =
+    `${base}\n\n` +
+    'The deployment may have been created under a different environment key ' +
+    '(for example evaluation-mode fallback to miso). Check controller logs for ' +
+    `effectiveEnvironmentKey, register '${requestedEnvKey}' in the controller, or run ` +
+    'aifabrix env deploy miso.';
+  return error;
+}
+
+function buildPollStatusError(pipelineRes, envRes) {
+  const terminal = [pipelineRes, envRes].filter(isTerminalPollApiResponse);
+  const source = terminal[0] || pipelineRes || envRes;
+  const message = source?.formattedError || source?.error || 'Unable to retrieve deployment status';
+  const err = new Error(message);
+  err.pollTerminal = true;
+  if (typeof source?.status === 'number') {
+    err.status = source.status;
+  }
+  return err;
+}
+
+/**
+ * Fetches deployment status by ID (pipeline endpoint first, then environments).
+ *
+ * @async
+ * @param {Function} getPipelineDeployment - pipeline.api getPipelineDeployment
+ * @param {Function} getDeployment - deployments.api getDeployment
+ * @param {string} controllerUrl
+ * @param {string} envKey
+ * @param {string} deploymentId
+ * @param {Object} apiAuthConfig
+ * @returns {Promise<Object|null>} Deployment record or null when status is not yet available (retry)
+ * @throws {Error} When both endpoints return a terminal HTTP error (e.g. environment not found)
+ */
+async function fetchDeploymentStatusById(
+  getPipelineDeployment,
+  getDeployment,
+  controllerUrl,
+  envKey,
+  deploymentId,
+  apiAuthConfig
+) {
+  let pipelineRes;
+  let envRes;
+
+  try {
+    pipelineRes = await getPipelineDeployment(controllerUrl, envKey, deploymentId, apiAuthConfig);
+    const fromPipeline = extractDeploymentRecord(pipelineRes);
+    if (fromPipeline) {
+      return fromPipeline;
+    }
+  } catch (err) {
+    pipelineRes = { success: false, error: err?.message || String(err) };
+  }
+
+  try {
+    envRes = await getDeployment(controllerUrl, envKey, deploymentId, apiAuthConfig);
+    const fromEnv = extractDeploymentRecord(envRes);
+    if (fromEnv) {
+      return fromEnv;
+    }
+  } catch (err) {
+    envRes = { success: false, error: err?.message || String(err) };
+  }
+
+  if (isTerminalPollApiResponse(pipelineRes) || isTerminalPollApiResponse(envRes)) {
+    throw buildPollStatusError(pipelineRes, envRes);
+  }
+
+  return null;
+}
+
+module.exports = {
+  TERMINAL_POLL_HTTP_STATUSES,
+  isTerminalPollApiResponse,
+  extractDeploymentRecord,
+  enrichPollTerminalError,
+  buildPollStatusError,
+  fetchDeploymentStatusById
+};

package/lib/deployment/environment.js

@@ -25,6 +25,7 @@ const {
   buildEnvironmentConfigFromPreset,
   loadAndValidateEnvironmentDeployConfig
 } = require('./environment-config');
+const { fetchDeploymentStatusById, enrichPollTerminalError } = require('./environment-poll-status');
 
 /**
  * Validates environment deployment prerequisites
@@ -161,31 +162,24 @@ async function sendEnvironmentDeployment(controllerUrl, envKey, authConfig, opti
 }
 
 /**
- * Fetches deployment status by ID (pipeline endpoint first, then environments)
- * Mirrors miso-controller manual test: GET pipeline/deployments/:id then GET environments/deployments/:id
+ * Fetches deployment status by ID (pipeline endpoint first, then environments).
  * @async
  * @param {string} controllerUrl - Controller URL
  * @param {string} envKey - Environment key
  * @param {string} deploymentId - Deployment ID
  * @param {Object} apiAuthConfig - Auth config { type: 'bearer', token }
- * @returns {Promise<Object|null>} Deployment record (status, progress, etc.) or null
+ * @returns {Promise<Object|null>} Deployment record or null when still in progress
+ * @throws {Error} On terminal poll errors (e.g. environment or deployment not found)
  */
 async function getDeploymentStatusById(controllerUrl, envKey, deploymentId, apiAuthConfig) {
-  try {
-    const pipelineRes = await getPipelineDeployment(controllerUrl, envKey, deploymentId, apiAuthConfig);
-    if (pipelineRes?.data?.data) return pipelineRes.data.data;
-    if (pipelineRes?.data && typeof pipelineRes.data === 'object' && pipelineRes.data.status) return pipelineRes.data;
-  } catch {
-    // Fallback to environments endpoint
-  }
-  try {
-    const envRes = await getDeployment(controllerUrl, envKey, deploymentId, apiAuthConfig);
-    if (envRes?.data?.data) return envRes.data.data;
-    if (envRes?.data && typeof envRes.data === 'object' && envRes.data.status) return envRes.data;
-  } catch {
-    // Ignore
-  }
-  return null;
+  return fetchDeploymentStatusById(
+    getPipelineDeployment,
+    getDeployment,
+    controllerUrl,
+    envKey,
+    deploymentId,
+    apiAuthConfig
+  );
 }
 
 /**
@@ -218,6 +212,33 @@ function processDeploymentStatusResponse(deployment, validatedEnvKey) {
   return null;
 }
 
+/**
+ * @param {number} attempt
+ * @param {number} maxAttempts
+ * @param {Object|null} deployment
+ */
+function logPollAttemptProgress(attempt, maxAttempts, deployment) {
+  const progress = deployment?.progress ?? 0;
+  const statusLabel = deployment?.status ?? 'pending';
+  logger.log(chalk.gray(`   Attempt ${attempt}/${maxAttempts}... Status: ${statusLabel} (${progress}%)`));
+}
+
+/**
+ * @param {Error} error
+ * @param {string} validatedEnvKey
+ * @throws {Error}
+ */
+function handlePollIterationError(error, validatedEnvKey) {
+  const msg = error?.message || '';
+  const isDeploymentFailed =
+    msg.includes('Environment deployment failed') ||
+    (msg.includes('failed') && !error.pollTerminal);
+  if (error.pollTerminal || isDeploymentFailed) {
+    throw enrichPollTerminalError(validatedEnvKey, error);
+  }
+  logger.log(chalk.yellow(`   ⚠ Status check failed (will retry): ${msg}`));
+}
+
 /**
  * Polls environment deployment status
  * @async
@@ -231,44 +252,31 @@ function processDeploymentStatusResponse(deployment, validatedEnvKey) {
 async function pollEnvironmentStatus(deploymentId, controllerUrl, envKey, authConfig, options = {}) {
   const validatedUrl = validateControllerUrl(controllerUrl);
   const validatedEnvKey = validateEnvironmentKey(envKey);
-
   const pollInterval = options.pollInterval || 5000;
   const maxAttempts = options.maxAttempts || 60;
+  const apiAuthConfig = { type: 'bearer', token: authConfig.token };
 
   logger.log(chalk.blue(`⏳ Polling environment status (${pollInterval}ms intervals)...`));
 
-  // Use centralized API client
-  const apiAuthConfig = { type: 'bearer', token: authConfig.token };
-
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
     try {
       await new Promise(resolve => setTimeout(resolve, pollInterval));
-
       const deployment = await getDeploymentStatusById(
         validatedUrl,
         validatedEnvKey,
         deploymentId,
         apiAuthConfig
       );
-      const progress = deployment?.progress ?? 0;
-      const statusLabel = deployment?.status ?? 'pending';
-      if (attempt <= maxAttempts) {
-        logger.log(chalk.gray(`   Attempt ${attempt}/${maxAttempts}... Status: ${statusLabel} (${progress}%)`));
-      }
+      logPollAttemptProgress(attempt, maxAttempts, deployment);
       const statusResult = processDeploymentStatusResponse(deployment, validatedEnvKey);
       if (statusResult) {
         return statusResult;
       }
     } catch (error) {
-      // If it's a terminal error (not a timeout), throw it
-      if (error.message && error.message.includes('failed')) {
-        throw error;
-      }
-      // Otherwise, continue polling
+      handlePollIterationError(error, validatedEnvKey);
     }
   }
 
-  // Timeout
   throw new Error(`Environment deployment timeout after ${maxAttempts} attempts. Check controller logs for status.`);
 }
 

package/lib/external-system/download-helpers.js

@@ -8,8 +8,90 @@
  * @version 2.0.0
  */
 
+const {
+  formatSuccessLine,
+  formatSuccessParagraph,
+  sectionTitle,
+  headerKeyValue
+} = require('../utils/cli-test-layout-chalk');
+const logger = require('../utils/logger');
 const { generateExternalReadmeContent } = require('../utils/external-readme');
 
+/**
+ * Validates system type from downloaded application
+ * @param {Object} application - External system configuration
+ * @returns {string} System type (openapi, mcp, custom)
+ */
+function validateSystemType(application) {
+  if (!application || typeof application !== 'object') {
+    throw new Error('Application configuration is required');
+  }
+  const validTypes = ['openapi', 'mcp', 'custom'];
+  const systemType = application.type;
+  if (!systemType || !validTypes.includes(systemType)) {
+    throw new Error(`Invalid system type: ${systemType}. Must be one of: ${validTypes.join(', ')}`);
+  }
+  return systemType;
+}
+
+function validateApplicationConfig(application) {
+  if (!application || typeof application !== 'object') {
+    throw new Error('Application configuration is required');
+  }
+  if (!application.key || typeof application.key !== 'string') {
+    throw new Error('Application key is required');
+  }
+}
+
+function validateDatasourceConfig(datasource, applicationKey) {
+  if (!datasource.key || typeof datasource.key !== 'string') {
+    throw new Error('Datasource key is required for all datasources');
+  }
+  if (!datasource.systemKey || typeof datasource.systemKey !== 'string') {
+    throw new Error('Datasource systemKey is required for all datasources');
+  }
+  if (datasource.systemKey !== applicationKey) {
+    throw new Error(
+      `Datasource systemKey (${datasource.systemKey}) does not match application key (${applicationKey})`
+    );
+  }
+}
+
+function validateDownloadedData(application, dataSources) {
+  validateApplicationConfig(application);
+  if (!Array.isArray(dataSources)) {
+    throw new Error('DataSources must be an array');
+  }
+  for (const datasource of dataSources) {
+    validateDatasourceConfig(datasource, application.key);
+  }
+}
+
+function logDownloadCommandHeader(systemKey) {
+  logger.log('');
+  logger.log(sectionTitle('Download'));
+  logger.log(headerKeyValue('System:', systemKey));
+}
+
+function displayDownloadSuccess(systemKey, finalPath, datasourceCount) {
+  logger.log(
+    formatSuccessParagraph(
+      `Downloaded ${systemKey} (${datasourceCount} datasource${datasourceCount === 1 ? '' : 's'})`
+    )
+  );
+  logger.log(headerKeyValue('Location:', finalPath));
+}
+
+async function runConvertToJsonIfRequested(systemKey) {
+  const { runConvert } = require('../commands/convert');
+  try {
+    await runConvert(systemKey, { format: 'json', force: true });
+    logger.log(formatSuccessLine('Converted component files to JSON'));
+  } catch (convertErr) {
+    throw new Error(`Download succeeded but convert to JSON failed: ${convertErr.message}`);
+  }
+}
+
 /**
  * Generates application.yaml content for downloaded system
  * @param {string} systemKey - System key
@@ -90,6 +172,11 @@ function generateReadme(systemKey, application, dataSources) {
 }
 
 module.exports = {
+  validateSystemType,
+  validateDownloadedData,
+  logDownloadCommandHeader,
+  displayDownloadSuccess,
+  runConvertToJsonIfRequested,
   generateVariablesYaml,
   generateReadme
 };