@aifabrix/builder 2.43.0 → 2.44.0

package/lib/external-system/deploy.js

@@ -1,8 +1,9 @@
+const { formatSuccessLine, formatSuccessParagraph } = require('../utils/cli-test-layout-chalk');
 /**
  * External System Deployment Module
  *
  * Handles deployment of external systems via controller pipeline.
- * Uses unified controller pipeline (same as regular apps) - no direct dataplane calls.
+ * After deploy, fetches dataplane list + system for readiness summary.
  *
  * @fileoverview External system deployment for AI Fabrix Builder
  * @author AI Fabrix Team
@@ -17,69 +18,116 @@ const { detectAppType } = require('../utils/paths');
 const { logOfflinePathWhenType } = require('../utils/cli-utils');
 const { resolveDataplaneUrl } = require('../utils/dataplane-resolver');
 const { getExternalSystem } = require('../api/external-systems.api');
+const { listDatasources } = require('../api/datasources-core.api');
+const { testSystemViaPipeline } = require('../api/pipeline.api');
+const { extractDatasources } = require('../datasource/list');
+const { unwrapApiData } = require('../utils/external-system-readiness-core');
+const { logDeployReadinessSummary } = require('../utils/external-system-readiness-deploy-display');
+const { parseControllerDeploymentOutcome } = require('../utils/controller-deployment-outcome');
 const { generateControllerManifest } = require('../generator/external-controller-manifest');
 const { validateExternalSystemComplete } = require('../validation/validate');
 const { displayValidationResults } = require('../validation/validate-display');
 
 /**
- * Displays API and MCP documentation URLs from dataplane when available
+ * Lists datasources for a system and loads system record for docs URLs.
  * @async
- * @function displayDeploymentDocs
  * @param {string} controllerUrl - Controller base URL
  * @param {string} environment - Environment key
- * @param {Object} authConfig - Authentication configuration
- * @param {string} systemKey - External system key
+ * @param {Object} authConfig - Auth config
+ * @param {string} systemKey - System key
+ * @returns {Promise<{ dataplaneUrl: string, datasources: Object[], system: Object|null, error: Error|null }>}
  */
-async function displayDeploymentDocs(controllerUrl, environment, authConfig, systemKey) {
+async function fetchDataplaneDeployReadiness(controllerUrl, environment, authConfig, systemKey) {
+  let dataplaneUrl;
   try {
-    const dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig);
-    const res = await getExternalSystem(dataplaneUrl, systemKey, authConfig);
-    const sys = res?.data || res;
-    if (!sys) return;
-
-    const apiDocumentUrl = sys.apiDocumentUrl;
-    const mcpServerUrl = sys.mcpServerUrl;
-    const openApiDocsPageUrl = sys.openApiDocsPageUrl;
-
-    const urls = [];
-    if (apiDocumentUrl && typeof apiDocumentUrl === 'string') {
-      urls.push({ label: 'API Docs', url: apiDocumentUrl });
-    }
-    if (mcpServerUrl && typeof mcpServerUrl === 'string') {
-      urls.push({ label: 'MCP Server', url: mcpServerUrl });
-    }
-    if (openApiDocsPageUrl && typeof openApiDocsPageUrl === 'string') {
-      urls.push({ label: 'OpenAPI Docs Page', url: openApiDocsPageUrl });
+    dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig);
+  } catch (err) {
+    return { dataplaneUrl: null, datasources: [], system: null, error: err };
+  }
+
+  let datasources = [];
+  try {
+    const listRes = await listDatasources(dataplaneUrl, authConfig, {
+      sourceSystemIdOrKey: systemKey,
+      pageSize: 100
+    });
+    if (listRes.success && listRes.data) {
+      try {
+        datasources = extractDatasources(listRes);
+      } catch (e) {
+        logger.log(
+          chalk.yellow(
+            `⚠ Unable to parse datasources list from dataplane response: ${e && e.message ? e.message : 'unknown error'}`
+          )
+        );
+        datasources = [];
+      }
     }
+  } catch (err) {
+    return { dataplaneUrl, datasources: [], system: null, error: err };
+  }
 
-    if (urls.length > 0) {
-      logger.log(chalk.blue('\nDocumentation:'));
-      urls.forEach(({ label, url }) => {
-        logger.log(chalk.blue(`   ${label}: ${url}`));
-      });
+  try {
+    const getRes = await getExternalSystem(dataplaneUrl, systemKey, authConfig);
+    const system = unwrapApiData(getRes);
+    return {
+      dataplaneUrl,
+      datasources,
+      system: system && typeof system === 'object' ? system : null,
+      error: null
+    };
+  } catch (err) {
+    if (datasources.length === 0) {
+      return { dataplaneUrl, datasources, system: null, error: err };
     }
-  } catch (_err) {
-    // Silently ignore: dataplane may be unreachable or docs not configured
+    return { dataplaneUrl, datasources, system: null, error: null };
   }
 }
 
 /**
- * Deploys via controller and displays success summary with docs
+ * Deploys via controller and prints readiness summary (config / deployment / runtime layers).
  * @async
- * @function executeDeployAndDisplay
  * @param {Object} manifest - Controller manifest
  * @param {string} controllerUrl - Controller base URL
  * @param {string} environment - Environment key
  * @param {Object} authConfig - Authentication configuration
- * @param {Object} options - Deployment options
+ * @param {Object} options - Deployment options (poll, probe, probeTimeout)
  * @returns {Promise<Object>} Deployment result
  */
+/**
+ * @param {Object} deploymentOutcome - from parseControllerDeploymentOutcome
+ */
+function logImmediateControllerDeploymentOutcome(deploymentOutcome) {
+  if (deploymentOutcome.ok) {
+    logger.log(formatSuccessParagraph('Controller deployment OK'));
+    return;
+  }
+  logger.log(chalk.red('\n✖ Controller deployment did not complete successfully'));
+  const parts = [deploymentOutcome.error, deploymentOutcome.message].filter(Boolean);
+  if (parts.length > 0) {
+    for (const line of parts) {
+      logger.log(chalk.red(`   ${line}`));
+    }
+    return;
+  }
+  if (deploymentOutcome.statusLabel) {
+    logger.log(
+      chalk.yellow(
+        `   Controller status: ${deploymentOutcome.statusLabel} (no message or error in API response)`
+      )
+    );
+    return;
+  }
+  logger.log(chalk.gray('   See Deployment section below for details.'));
+}
+
 async function executeDeployAndDisplay(manifest, controllerUrl, environment, authConfig, options) {
   const deployer = require('../deployment/deployer');
   const pollOpts = {
     poll: options.poll,
     pollInterval: options.pollInterval !== undefined ? options.pollInterval : 500,
-    pollMaxAttempts: options.pollMaxAttempts,
+    // External-system deploy only: ~10s poll window (matches controller→dataplane publish timeout). Not used for Azure/webapp deploy polling.
+    pollMaxAttempts: options.pollMaxAttempts !== undefined ? options.pollMaxAttempts : 22,
     ...options
   };
   const result = await deployer.deployToController(
@@ -89,17 +137,54 @@ async function executeDeployAndDisplay(manifest, controllerUrl, environment, aut
     authConfig,
     pollOpts
   );
-  logger.log(chalk.green('\n✅ External system deployed successfully!'));
-  logger.log(chalk.blue(`System: ${manifest.key}`));
-  logger.log(chalk.blue(`Datasources: ${manifest.dataSources.length}`));
-  await displayDeploymentDocs(controllerUrl, environment, authConfig, manifest.key);
+
+  const deploymentOutcome = parseControllerDeploymentOutcome(result);
+  logImmediateControllerDeploymentOutcome(deploymentOutcome);
+
+  const ctx = await fetchDataplaneDeployReadiness(
+    controllerUrl,
+    environment,
+    authConfig,
+    manifest.key
+  );
+
+  let probeData = null;
+  if (options.probe && ctx.dataplaneUrl && !ctx.error) {
+    logger.log(chalk.blue('\nRunning runtime checks (--probe)...'));
+    try {
+      const pr = await testSystemViaPipeline(ctx.dataplaneUrl, manifest.key, authConfig, {}, {
+        timeout: options.probeTimeout || 120000
+      });
+      if (pr.success === false) {
+        logger.log(
+          chalk.yellow(`⚠ Probe request failed: ${pr.formattedError || pr.error || 'unknown error'}`)
+        );
+      } else {
+        probeData = unwrapApiData(pr);
+      }
+    } catch (e) {
+      logger.log(chalk.yellow(`⚠ Probe failed: ${e.message}`));
+    }
+  }
+
+  logDeployReadinessSummary({
+    environment,
+    dataplaneUrl: ctx.dataplaneUrl,
+    manifest,
+    datasources: ctx.datasources,
+    systemFromDataplane: ctx.system,
+    fetchError: ctx.error,
+    deploymentOk: deploymentOutcome.ok,
+    deploymentDetail: deploymentOutcome.ok ? null : deploymentOutcome,
+    probeData
+  });
+
   return result;
 }
 
 /**
  * Prepares deployment configuration (auth, controller URL, environment)
  * @async
- * @function prepareDeploymentConfig
  * @param {string} appName - Application name
  * @param {Object} options - Deployment options
  * @returns {Promise<Object>} Deployment configuration
@@ -119,16 +204,12 @@ async function prepareDeploymentConfig(appName, _options) {
 
 /**
  * Deploys external system via controller pipeline (same as regular apps)
- * Uses unified controller pipeline - no direct dataplane calls
  *
  * @async
- * @function deployExternalSystem
  * @param {string} appName - Application name
  * @param {Object} options - Deployment options
- * @param {string} [options.environment] - Environment (dev, tst, pro)
- * @param {string} [options.controller] - Controller URL
- * @param {boolean} [options.poll] - Poll for deployment status
- * @param {number} [options.pollInterval] - Polling interval in milliseconds (default: 500ms for external systems)
+ * @param {boolean} [options.probe] - Run dataplane validation/run after deploy
+ * @param {number} [options.probeTimeout] - Probe timeout ms (default 120000)
  * @returns {Promise<Object>} Deployment result
  * @throws {Error} If deployment fails
  */
@@ -139,7 +220,6 @@ async function deployExternalSystem(appName, options = {}) {
 
     logger.log(chalk.blue(`\n🚀 Deploying external system: ${appName}`));
 
-    // Step 0: Validate before deployment (same as validate command)
     logger.log(chalk.blue('🔍 Validating external system before deployment...'));
     const validationResult = await validateExternalSystemComplete(appName, options);
 
@@ -148,15 +228,12 @@ async function deployExternalSystem(appName, options = {}) {
       throw new Error('Validation failed. Fix errors before deploying.');
     }
 
-    logger.log(chalk.green('✓ Validation passed, proceeding with deployment...'));
+    logger.log(formatSuccessLine('Local validation passed, proceeding with deployment...'));
 
-    // Step 1: Generate controller manifest (validated, ready for deployment)
     const manifest = await generateControllerManifest(appName, options);
 
-    // Step 2: Get deployment configuration (auth, controller URL, etc.)
     const { environment, controllerUrl, authConfig } = await prepareDeploymentConfig(appName, options);
 
-    // Step 3: Deploy via controller pipeline (same as regular apps)
     const result = await executeDeployAndDisplay(
       manifest,
       controllerUrl,
@@ -175,6 +252,7 @@ async function deployExternalSystem(appName, options = {}) {
 }
 
 module.exports = {
-  deployExternalSystem
+  deployExternalSystem,
+  executeDeployAndDisplay,
+  fetchDataplaneDeployReadiness
 };
-

package/lib/external-system/download-helpers.js

@@ -73,7 +73,7 @@ function generateReadme(systemKey, application, dataSources) {
     return {
       entityType: datasourceKeyOnly,
       displayName: ds.displayName || ds.name || ds.key || `Datasource ${index + 1}`,
-      fileName: `${systemKey}-datasource-${datasourceKeyOnly}.yaml`
+      fileName: `${systemKey}-datasource-${datasourceKeyOnly}.json`
     };
   });
 

package/lib/external-system/download.js

@@ -1,3 +1,4 @@
+const { formatSuccessLine, formatSuccessParagraph } = require('../utils/cli-test-layout-chalk');
 /**
  * External System Download Module
  *
@@ -129,7 +130,7 @@ async function setupAuthenticationAndDataplane(systemKey, _options, _config) {
   const { resolveDataplaneUrl } = require('../utils/dataplane-resolver');
   logger.log(chalk.blue('🌐 Resolving dataplane URL...'));
   const dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig);
-  logger.log(chalk.green(`✓ Dataplane URL: ${dataplaneUrl}`));
+  logger.log(formatSuccessLine(`Dataplane URL: ${dataplaneUrl}`));
 
   return { authConfig, dataplaneUrl };
 }
@@ -287,8 +288,8 @@ function validateAndLogDownloadedData(application, dataSources) {
   logger.log(chalk.blue('🔍 Validating downloaded data...'));
   validateDownloadedData(application, dataSources);
   const systemType = validateSystemType(application);
-  logger.log(chalk.green(`✓ System type: ${systemType}`));
-  logger.log(chalk.green(`✓ Found ${dataSources.length} datasource(s)`));
+  logger.log(formatSuccessLine(`System type: ${systemType}`));
+  logger.log(formatSuccessLine(`Found ${dataSources.length} datasource(s)`));
   return systemType;
 }
 
@@ -367,7 +368,7 @@ async function processDownloadedSystem(systemKey, manifest, splitOptions = {}) {
   const deployJson = buildDeployJsonFromManifest(application, dataSources, version);
   const deployJsonPath = path.join(finalPath, `${systemKey}-deploy.json`);
   await fs.writeFile(deployJsonPath, JSON.stringify(deployJson, null, 2), 'utf8');
-  logger.log(chalk.green(`✓ Created: ${path.relative(process.cwd(), deployJsonPath)}`));
+  logger.log(formatSuccessLine(`Created: ${path.relative(process.cwd(), deployJsonPath)}`));
 
   logger.log(chalk.blue('📂 Splitting deploy JSON into component files...'));
   const splitResult = await generator.splitDeployJson(deployJsonPath, finalPath, splitOptions);
@@ -390,7 +391,7 @@ async function processDownloadedSystem(systemKey, manifest, splitOptions = {}) {
  * @param {number} datasourceCount - Number of datasources
  */
 function displayDownloadSuccess(systemKey, finalPath, datasourceCount) {
-  logger.log(chalk.green('\n✅ External system downloaded successfully!'));
+  logger.log(formatSuccessParagraph('External system downloaded successfully!'));
   logger.log(chalk.blue(`Location: ${finalPath}`));
   logger.log(chalk.blue(`System: ${systemKey}`));
   logger.log(chalk.blue(`Datasources: ${datasourceCount}`));
@@ -414,7 +415,7 @@ async function runConvertToJsonIfRequested(systemKey) {
   const { runConvert } = require('../commands/convert');
   try {
     await runConvert(systemKey, { format: 'json', force: true });
-    logger.log(chalk.green('✓ Converted component files to JSON'));
+    logger.log(formatSuccessLine('Converted component files to JSON'));
   } catch (convertErr) {
     throw new Error(`Download succeeded but convert to JSON failed: ${convertErr.message}`);
   }

package/lib/external-system/generator.js

@@ -9,10 +9,10 @@
  * @version 2.0.0
  */
 
+const { formatSuccessLine } = require('../utils/cli-test-layout-chalk');
 const fs = require('fs').promises;
 const path = require('path');
 const handlebars = require('handlebars');
-const chalk = require('chalk');
 const logger = require('../utils/logger');
 const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
 const { loadConfigFile, writeConfigFile } = require('../utils/config-format');
@@ -135,6 +135,68 @@ function resourceTypeToSchemaEntityType(resourceType) {
   return resourceType === 'document' ? 'documentStorage' : 'recordStorage';
 }
 
+/**
+ * Build { dimKey, field } entries for root dimensions (v2.4+ dimensionBinding).
+ * @param {Object} dimensions - Map of dimension key → attribute path (e.g. metadata.country)
+ * @returns {Array<{ dimKey: string, field: string }>}
+ */
+function buildDimensionBindingEntries(dimensions) {
+  if (!dimensions || typeof dimensions !== 'object' || Array.isArray(dimensions)) return [];
+  return Object.entries(dimensions).map(([dimKey, path]) => {
+    const field = typeof path === 'string' && path.startsWith('metadata.')
+      ? path.slice('metadata.'.length).trim()
+      : dimKey;
+    return { dimKey, field };
+  });
+}
+
+/**
+ * @param {string} schemaEntityType - Resolved schema entityType
+ * @returns {boolean}
+ */
+function isStorageEntityType(schemaEntityType) {
+  return schemaEntityType === 'recordStorage' || schemaEntityType === 'documentStorage';
+}
+
+/**
+ * Puts externalId first for storage entity types when present (schema v2.4 join identity).
+ * @param {string[]} keys - Attribute / metadata property names
+ * @param {string} schemaEntityType - Resolved schema entityType
+ * @returns {string[]}
+ */
+function orderMetadataAttributeKeys(keys, schemaEntityType) {
+  const out = [...keys];
+  if (isStorageEntityType(schemaEntityType) && out.includes('externalId')) {
+    return ['externalId', ...out.filter(k => k !== 'externalId')];
+  }
+  return out;
+}
+
+/**
+ * Default fieldMappings.attributes + metadata key order when the wizard/config omits attributes.
+ * @param {string} schemaEntityType - Resolved schema entityType
+ * @returns {{ merged: Object, keys: string[] }}
+ */
+function defaultAttributesForEntityType(schemaEntityType) {
+  if (isStorageEntityType(schemaEntityType)) {
+    return {
+      merged: {
+        externalId: { expression: '{{raw.id}}' },
+        id: { expression: '{{raw.id}}' },
+        name: { expression: '{{raw.name}}' }
+      },
+      keys: ['externalId', 'id', 'name']
+    };
+  }
+  return {
+    merged: {
+      id: { expression: '{{raw.id}}' },
+      name: { expression: '{{raw.name}}' }
+    },
+    keys: ['id', 'name']
+  };
+}
+
 /**
  * Build datasource context object for template rendering
  * @param {Object} opts - Options
@@ -151,9 +213,31 @@ function resourceTypeToSchemaEntityType(resourceType) {
 function buildDatasourceContext({ config, datasourceKey, dimensions, attributes, fullDatasourceKey, entityKey, schemaEntityType, resourceType }) {
   const displayName = config.datasourceDisplayName || datasourceKey.replace(/-/g, ' ').replace(/\b\w/g, l => l.toUpperCase());
   const description = config.datasourceDescription || `External datasource for ${datasourceKey}`;
+  const attrsIn = attributes && typeof attributes === 'object' && !Array.isArray(attributes) ? attributes : {};
+  let mergedAttributes = { ...attrsIn };
+
+  let attributeKeysForMetadata;
+  if (Object.keys(mergedAttributes).length > 0) {
+    attributeKeysForMetadata = orderMetadataAttributeKeys(Object.keys(mergedAttributes), schemaEntityType);
+  } else {
+    const defaults = defaultAttributesForEntityType(schemaEntityType);
+    mergedAttributes = defaults.merged;
+    attributeKeysForMetadata = defaults.keys;
+  }
+
+  if (isStorageEntityType(schemaEntityType) && !mergedAttributes.externalId) {
+    mergedAttributes.externalId = { expression: '{{raw.id}}' };
+    attributeKeysForMetadata = orderMetadataAttributeKeys(Object.keys(mergedAttributes), schemaEntityType);
+  }
+
   const primaryKey = Array.isArray(config.primaryKey) && config.primaryKey.length > 0
     ? config.primaryKey
-    : ['id'];
+    : (attributeKeysForMetadata.includes('id') ? ['id'] : [attributeKeysForMetadata[0]]);
+  const labelKey = Array.isArray(config.labelKey) && config.labelKey.length > 0
+    ? config.labelKey
+    : attributeKeysForMetadata;
+  const dimensionBindingEntries = buildDimensionBindingEntries(dimensions);
+
   return {
     fullDatasourceKey,
     entityKey,
@@ -163,9 +247,11 @@ function buildDatasourceContext({ config, datasourceKey, dimensions, attributes,
     schemaEntityType,
     resourceType,
     primaryKey,
+    labelKey,
+    attributeKeysForMetadata,
+    dimensionBindingEntries,
     systemType: config.systemType || 'openapi',
-    dimensions: Object.keys(dimensions).length > 0 ? dimensions : null,
-    attributes: Object.keys(attributes).length > 0 ? attributes : null,
+    attributes: mergedAttributes,
     raw: { id: '{{raw.id}}', name: '{{raw.name}}' }
   };
 }
@@ -250,7 +336,7 @@ async function generateExternalSystemFiles(appPath, appName, config, format = 'y
 
     // Generate external system file
     const systemPath = await generateExternalSystemTemplate(appPath, systemKey, config, fmt);
-    logger.log(chalk.green(`✓ Generated external system: ${path.basename(systemPath)}`));
+    logger.log(formatSuccessLine(`Generated external system: ${path.basename(systemPath)}`));
 
     // Generate datasource JSON files
     const datasourcePaths = [];
@@ -278,7 +364,7 @@ async function generateExternalSystemFiles(appPath, appName, config, format = 'y
 
       const datasourcePath = await generateExternalDataSourceTemplate(appPath, datasourceKey, datasourceConfig, fmt);
       datasourcePaths.push(datasourcePath);
-      logger.log(chalk.green(`✓ Generated datasource: ${path.basename(datasourcePath)}`));
+      logger.log(formatSuccessLine(`Generated datasource: ${path.basename(datasourcePath)}`));
     }
 
     // Update application config with externalIntegration block

package/lib/external-system/integration-test-dispatch.js

@@ -0,0 +1,26 @@
+/**
+ * @fileoverview Dispatch rules for external test-integration (system vs per-datasource POST /validation/run).
+ */
+
+'use strict';
+
+/**
+ * Use one externalSystem-scoped validation run when multiple datasources exist (default).
+ * Per-datasource runs when {@code options.perDatasource}, a datasource filter, or custom payload is set.
+ *
+ * @param {Object} options - CLI options
+ * @param {string} [options.datasource] - Single-datasource filter
+ * @param {boolean} [options.perDatasource] - Force one POST per datasource (externalDataSource scope)
+ * @param {Array<{data?: Object}>} datasourcesToTest - Entries from manifest
+ * @param {*} customPayload - Parsed payload from {@code --payload}, or null
+ * @returns {boolean}
+ */
+function shouldUseSystemLevelIntegrationCall(options, datasourcesToTest, customPayload) {
+  const noDatasourceFilter = !options || !options.datasource;
+  const multi = Array.isArray(datasourcesToTest) && datasourcesToTest.length > 1;
+  const noCustomPayload = customPayload === null || customPayload === undefined;
+  const forcePerDs = options && options.perDatasource === true;
+  return noDatasourceFilter && multi && noCustomPayload && !forcePerDs;
+}
+
+module.exports = { shouldUseSystemLevelIntegrationCall };

package/lib/external-system/test-execution.js

@@ -11,6 +11,7 @@
 const chalk = require('chalk');
 const logger = require('../utils/logger');
 const testHelpers = require('../utils/external-system-test-helpers');
+const { infoLine } = require('../utils/cli-test-layout-chalk');
 
 /**
  * Executes test for a single datasource
@@ -62,7 +63,10 @@ async function testSingleDatasourceIntegration(datasourceFile, systemKey, datapl
   const datasource = datasourceFile.data;
   const datasourceKey = datasource.key;
 
-  logger.log(chalk.blue(`\n📡 Testing datasource: ${datasourceKey}`));
+  if (options.verbose) {
+    logger.log('');
+    logger.log(infoLine(`📡 Testing datasource: ${datasourceKey}`));
+  }
 
   // Determine payload to use
   const payloadTemplate = testHelpers.determinePayloadTemplate(datasource, datasourceKey, customPayload);

package/lib/external-system/test-helpers.js

@@ -8,8 +8,6 @@
  * @version 2.0.0
  */
 
-const chalk = require('chalk');
-const logger = require('../utils/logger');
 const externalSystemSchema = require('../schema/external-system.schema.json');
 const { validateAgainstSchema } = require('../utils/external-system-validators');
 
@@ -63,7 +61,6 @@ function validateSystemFiles(systemFiles, schema) {
  * @param {Object} results - Test results object
  */
 function validateSystemFilesForTest(systemFiles, results) {
-  logger.log(chalk.blue('📋 Validating system files...'));
   const systemValidation = validateSystemFiles(systemFiles, externalSystemSchema);
   results.valid = systemValidation.valid;
   results.errors.push(...systemValidation.errors);
@@ -81,7 +78,6 @@ function validateSystemFilesForTest(systemFiles, results) {
  * @param {Function} determineDatasourcesToTest - Function to determine datasources to test
  */
 function validateDatasourceFilesForTest(datasourceFiles, systemFiles, results, options, validateSingleDatasource, determineDatasourcesToTest) {
-  logger.log(chalk.blue('📋 Validating datasource files...'));
   const datasourcesToTest = determineDatasourcesToTest(datasourceFiles, options.datasource);
   const systemKey = systemFiles.length > 0 ? systemFiles[0].data.key : null;
 

package/lib/external-system/test-system-level-helpers.js

@@ -0,0 +1,110 @@
+/**
+ * Helpers for system-level pipeline test mapping (keeps runSystemLevelTest under max-lines-per-function).
+ * @fileoverview
+ */
+'use strict';
+
+const { integrationResultFromEnvelope } = require('../utils/datasource-test-run-legacy-adapter');
+
+/**
+ * Build a single failure-shaped datasource result row.
+ * @param {string} error
+ * @returns {Object}
+ */
+function systemFailureRow(error) {
+  return {
+    key: 'system',
+    success: false,
+    skipped: false,
+    validationResults: {},
+    fieldMappingResults: {},
+    endpointTestResults: {},
+    error
+  };
+}
+
+/**
+ * Map unified envelope-style response to datasourceResults.
+ * @param {Object} data
+ * @param {string} datasourceKey
+ * @returns {{ success: boolean, datasourceResults: Object[] }}
+ */
+function mapEnvelopeSingleDatasource(data, datasourceKey) {
+  const legacy = integrationResultFromEnvelope(data, datasourceKey);
+  return {
+    success: legacy.success,
+    datasourceResults: [
+      {
+        key: legacy.key,
+        success: legacy.success,
+        skipped: legacy.skipped,
+        validationResults: legacy.validationResults,
+        fieldMappingResults: legacy.fieldMappingResults,
+        endpointTestResults: legacy.endpointTestResults,
+        error: legacy.error,
+        envelope: legacy.envelope
+      }
+    ]
+  };
+}
+
+/**
+ * Map legacy array-shaped results to normalized datasource rows.
+ * @param {Array} rawResults
+ * @param {Object} data
+ * @returns {{ success: boolean, datasourceResults: Object[] }}
+ */
+function mapLegacyRawResults(rawResults, data) {
+  const datasourceResults = [];
+  let success = true;
+
+  for (const r of rawResults) {
+    const dsKey = r.key || r.sourceKey || r.name || r.datasourceKey;
+    const dsResult = {
+      key: dsKey,
+      success: r.success !== false,
+      skipped: !!r.skipped,
+      reason: r.reason,
+      validationResults: r.validationResults || {},
+      fieldMappingResults: r.fieldMappingResults || {},
+      endpointTestResults: r.endpointTestResults || {}
+    };
+    if (r.error) dsResult.error = r.error;
+    if (!dsResult.success && !dsResult.skipped) success = false;
+    datasourceResults.push(dsResult);
+  }
+
+  if (rawResults.length === 0 && data.success === false) {
+    success = false;
+    datasourceResults.push(
+      systemFailureRow(data.error || data.formattedError || 'System test failed')
+    );
+  }
+
+  return { success, datasourceResults };
+}
+
+/**
+ * Normalize pipeline response body to { success, datasourceResults }.
+ * @param {Object} data
+ * @returns {{ success: boolean, datasourceResults: Object[] }}
+ */
+function mapPipelineDataToDatasourceResults(data) {
+  if (
+    data &&
+    typeof data === 'object' &&
+    typeof data.status === 'string' &&
+    typeof data.datasourceKey === 'string'
+  ) {
+    return mapEnvelopeSingleDatasource(data, data.datasourceKey);
+  }
+
+  const rawResults =
+    data.datasourceResults || data.results || data.data?.datasourceResults || (Array.isArray(data) ? data : []);
+  return mapLegacyRawResults(rawResults, data);
+}
+
+module.exports = {
+  systemFailureRow,
+  mapPipelineDataToDatasourceResults
+};