@aifabrix/builder 2.41.0 → 2.42.0

package/lib/core/config.js

@@ -126,7 +126,8 @@ function getDefaultConfig() {
     environment: 'dev',
     controller: undefined,
     environments: {},
-    device: {}
+    device: {},
+    format: undefined
   };
 }
 
@@ -489,4 +490,9 @@ const { createPathConfigFunctions } = require('../utils/config-paths');
 const pathConfigFunctions = createPathConfigFunctions(getConfig, saveConfig);
 Object.assign(exportsObj, pathConfigFunctions);
 
+// Format preference functions
+const { createFormatFunctions } = require('../utils/config-format-preference');
+const formatFunctions = createFormatFunctions(getConfig, saveConfig);
+Object.assign(exportsObj, formatFunctions);
+
 module.exports = exportsObj;

package/lib/core/secrets.js

@@ -206,6 +206,38 @@ async function loadMergedConfigAndUserSecrets() {
   }
 }
 
+/**
+ * Loads merged secrets using config/user cascade, builder file merge, and default fallback.
+ * @async
+ * @returns {Promise<Object>} Merged secrets object (not decrypted)
+ */
+async function loadSecretsWithFallbacks() {
+  let merged = await loadMergedConfigAndUserSecrets();
+  if (!merged || Object.keys(merged).length === 0) {
+    merged = loadPrimaryUserSecrets();
+    if (Object.keys(merged).length === 0) {
+      merged = loadUserSecrets();
+    }
+    merged = await applyCanonicalSecretsOverride(merged);
+  }
+  try {
+    const projectRoot = pathsUtil.getProjectRoot();
+    if (projectRoot) {
+      const builderPath = path.join(projectRoot, 'builder', 'secrets.local.yaml');
+      if (fs.existsSync(builderPath)) {
+        const builderSecrets = mergeUserWithConfigFile(merged || {}, builderPath);
+        if (builderSecrets) merged = builderSecrets;
+      }
+    }
+  } catch {
+    // Ignore (e.g. no project root or read error)
+  }
+  if (Object.keys(merged).length === 0) {
+    merged = loadDefaultSecrets();
+  }
+  return merged;
+}
+
 async function loadSecrets(secretsPath, _appName) {
   if (secretsPath) {
     const resolvedPath = resolveSecretsPath(secretsPath);
@@ -218,18 +250,7 @@ async function loadSecrets(secretsPath, _appName) {
     }
     return await decryptSecretsObject(explicitSecrets);
   }
-
-  let mergedSecrets = await loadMergedConfigAndUserSecrets();
-  if (!mergedSecrets || Object.keys(mergedSecrets).length === 0) {
-    mergedSecrets = loadPrimaryUserSecrets();
-    if (Object.keys(mergedSecrets).length === 0) {
-      mergedSecrets = loadUserSecrets();
-    }
-    mergedSecrets = await applyCanonicalSecretsOverride(mergedSecrets);
-  }
-  if (Object.keys(mergedSecrets).length === 0) {
-    mergedSecrets = loadDefaultSecrets();
-  }
+  const mergedSecrets = await loadSecretsWithFallbacks();
   ensureNonEmptySecrets(mergedSecrets);
   return await decryptSecretsObject(mergedSecrets);
 }

package/lib/datasource/deploy.js

@@ -16,6 +16,11 @@ const { getEnvironmentApplication } = require('../api/environments.api');
 const { publishDatasourceViaPipeline } = require('../api/pipeline.api');
 const { formatApiError } = require('../utils/api-error-handler');
 const logger = require('../utils/logger');
+const { logDataplanePipelineWarning } = require('../utils/dataplane-pipeline-warning');
+const {
+  buildResolvedEnvMapForIntegration,
+  resolveConfigurationValues
+} = require('../utils/configuration-env-resolver');
 const { validateDatasourceFile } = require('./validate');
 
 /**
@@ -50,7 +55,7 @@ async function getDataplaneUrl(controllerUrl, appKey, environment, authConfig) {
   if (!dataplaneUrl) {
     const appType = application.configuration?.type || application.type;
     if (appType === 'external') {
-      throw new Error('Dataplane URL not found for external system. Provide --dataplane <url>.');
+      throw new Error('Dataplane URL not found for external system in application configuration.');
     }
     throw new Error('Dataplane URL not found in application configuration');
   }
@@ -107,8 +112,6 @@ async function validateAndLoadDatasourceFile(filePath) {
  * @param {string} controllerUrl - Controller URL
  * @param {string} environment - Environment key
  * @param {string} appKey - Application key
- * @param {Object} [options] - Options
- * @param {string} [options.dataplane] - Dataplane URL override
  * @returns {Promise<Object>} Object with authConfig and dataplaneUrl
  */
 async function setupDeploymentAuth(controllerUrl, environment, appKey) {
@@ -154,6 +157,7 @@ async function setupDeploymentAuth(controllerUrl, environment, appKey) {
 async function publishDatasourceToDataplane(dataplaneUrl, systemKey, authConfig, datasourceConfig) {
   requireBearerForDataplanePipeline(authConfig);
   logger.log(chalk.blue('\n🚀 Publishing datasource to dataplane...'));
+  logDataplanePipelineWarning();
 
   const publishResponse = await publishDatasourceViaPipeline(dataplaneUrl, systemKey, authConfig, datasourceConfig);
 
@@ -228,6 +232,11 @@ async function deployDatasource(appKey, filePath, _options) {
     throw new Error('systemKey is required in datasource configuration');
   }
 
+  if (Array.isArray(datasourceConfig.configuration) && datasourceConfig.configuration.length > 0) {
+    const { envMap, secrets } = await buildResolvedEnvMapForIntegration(systemKey);
+    resolveConfigurationValues(datasourceConfig.configuration, envMap, secrets, systemKey);
+  }
+
   // Setup authentication and get dataplane URL
   const { authConfig, dataplaneUrl } = await setupDeploymentAuth(controllerUrl, environment, appKey);
 

package/lib/datasource/test-e2e.js

@@ -0,0 +1,219 @@
+/**
+ * Datasource E2E test - run full E2E test via dataplane external API
+ * @fileoverview Datasource E2E test logic (config, credential, sync, data, CIP)
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+/* eslint-disable max-statements -- Auth setup, API call, polling, debug log */
+
+const path = require('path');
+const fs = require('fs').promises;
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const { getIntegrationPath, resolveIntegrationAppKeyFromCwd } = require('../utils/paths');
+const { resolveDataplaneUrl } = require('../utils/dataplane-resolver');
+const { resolveControllerUrl } = require('../utils/controller-url');
+const { getDeviceOnlyAuth } = require('../utils/token-manager');
+const { testDatasourceE2E, getE2ETestRun } = require('../api/external-test.api');
+const { writeTestLog } = require('../utils/test-log-writer');
+
+const DEFAULT_POLL_INTERVAL_MS = 2500;
+const DEFAULT_POLL_TIMEOUT_MS = 15 * 60 * 1000;
+
+/**
+ * Resolve appKey for datasource test-e2e
+ * @param {string} [appKey] - Explicit app key from --app
+ * @returns {string}
+ */
+function resolveAppKey(appKey) {
+  if (appKey) return appKey;
+  const fromCwd = resolveIntegrationAppKeyFromCwd();
+  if (fromCwd) return fromCwd;
+  throw new Error(
+    'Could not determine app context. Use --app <appKey> or run from integration/<appKey>/ directory.'
+  );
+}
+
+/**
+ * Resolve primaryKeyValue for request body: string as-is, or read and parse JSON from @path
+ * @param {string} [value] - Literal value or path prefixed with @ (e.g. @pk.json)
+ * @returns {Promise<string|Object|null>} Resolved value for body.primaryKeyValue, or null if absent
+ */
+async function resolvePrimaryKeyValue(value) {
+  if (value === null || value === undefined || value === '') return null;
+  const str = String(value).trim();
+  if (str.startsWith('@')) {
+    const filePath = path.resolve(str.slice(1).trim());
+    const content = await fs.readFile(filePath, 'utf8');
+    return JSON.parse(content);
+  }
+  return str;
+}
+
+/**
+ * Build E2E request body from options
+ * @param {Object} options - Command options
+ * @returns {Promise<Object>} Request body
+ */
+async function buildE2EBody(options) {
+  const body = {};
+  if (options.debug) body.includeDebug = true;
+  if (options.testCrud === true) body.testCrud = true;
+  if (options.recordId !== undefined && options.recordId !== null && options.recordId !== '') body.recordId = String(options.recordId);
+  if (options.cleanup === false) body.cleanup = false;
+  else if (options.cleanup === true) body.cleanup = true;
+  const pk = await resolvePrimaryKeyValue(options.primaryKeyValue);
+  if (pk !== null && pk !== undefined) body.primaryKeyValue = pk;
+  return body;
+}
+
+/**
+ * Poll E2E test run until completed or failed
+ * @param {string} dataplaneUrl - Dataplane URL
+ * @param {string} sourceIdOrKey - Source ID or key
+ * @param {string} testRunId - Test run ID
+ * @param {Object} authConfig - Auth config
+ * @param {Object} opts - Poll options
+ * @param {number} [opts.intervalMs] - Poll interval (ms)
+ * @param {number} [opts.timeoutMs] - Max wait (ms)
+ * @param {boolean} [opts.verbose] - Log each poll
+ * @returns {Promise<Object>} Final poll result (status completed or failed)
+ */
+async function pollE2ETestRun(dataplaneUrl, sourceIdOrKey, testRunId, authConfig, opts = {}) {
+  const intervalMs = opts.intervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_POLL_TIMEOUT_MS;
+  const verbose = opts.verbose === true;
+  const deadline = Date.now() + timeoutMs;
+  let last;
+  while (Date.now() < deadline) {
+    last = await getE2ETestRun(dataplaneUrl, sourceIdOrKey, testRunId, authConfig);
+    if (last.status === 'completed' || last.status === 'failed') {
+      return last;
+    }
+    if (verbose) {
+      const steps = last.completedActions || [];
+      logger.log(chalk.gray(`  Polling… status: ${last.status}, ${steps.length} step(s) completed`));
+    }
+    await new Promise(r => setTimeout(r, intervalMs));
+  }
+  throw new Error(
+    `E2E test run did not complete within ${timeoutMs / 1000}s (run ID: ${testRunId})`
+  );
+}
+
+/**
+ * Run E2E test for one datasource (Bearer token or API key required; no client credentials).
+ * Default: async start + polling until completed/failed. Use options.async === false for sync.
+ *
+ * @async
+ * @param {string} datasourceKey - Datasource key (used as sourceIdOrKey)
+ * @param {Object} options - Options
+ * @param {string} [options.app] - App key (or resolve from cwd)
+ * @param {string} [options.environment] - Environment (dev, tst, pro)
+ * @param {boolean} [options.debug] - Include debug, write log file
+ * @param {boolean} [options.verbose] - Verbose output (e.g. poll progress)
+ * @param {boolean} [options.async] - If false, use sync mode (no polling). Default true.
+ * @param {boolean} [options.testCrud] - Set body testCrud true
+ * @param {string} [options.recordId] - Set body recordId
+ * @param {boolean} [options.cleanup] - Set body cleanup (default true)
+ * @param {string} [options.primaryKeyValue] - Set body primaryKeyValue (string or @path to JSON)
+ * @param {number} [options.pollIntervalMs] - Poll interval in ms (default 2500)
+ * @param {number} [options.pollTimeoutMs] - Poll timeout in ms (default 15 min)
+ * @returns {Promise<Object>} E2E test result (steps, success, error, etc.)
+ */
+async function runDatasourceTestE2E(datasourceKey, options = {}) {
+  if (!datasourceKey || typeof datasourceKey !== 'string') {
+    throw new Error('Datasource key is required');
+  }
+  const appKey = resolveAppKey(options.app);
+  const controllerUrl = await resolveControllerUrl();
+  const { resolveEnvironment } = require('../core/config');
+  const environment = options.environment || await resolveEnvironment();
+  const authConfig = await getDeviceOnlyAuth(controllerUrl);
+  const dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig);
+
+  logger.log(chalk.blue(`\n🧪 Running E2E test for datasource: ${datasourceKey}`));
+
+  const body = await buildE2EBody(options);
+  const useAsync = options.async !== false;
+  const requestMeta = {
+    sourceIdOrKey: datasourceKey,
+    includeDebug: options.debug,
+    testCrud: options.testCrud,
+    recordId: options.recordId,
+    cleanup: options.cleanup,
+    primaryKeyValue: options.primaryKeyValue !== undefined && options.primaryKeyValue !== null
+  };
+
+  const execOpts = {
+    dataplaneUrl,
+    datasourceKey,
+    authConfig,
+    body,
+    useAsync,
+    verbose: options.verbose,
+    pollIntervalMs: options.pollIntervalMs,
+    pollTimeoutMs: options.pollTimeoutMs
+  };
+  let data;
+  try {
+    data = await executeE2EWithOptionalPoll(execOpts);
+  } catch (error) {
+    if (options.debug) {
+      const appPath = getIntegrationPath(appKey);
+      const integrationDir = path.dirname(appPath);
+      await writeTestLog(appKey, { request: requestMeta, error: error.message }, 'test-e2e', integrationDir);
+    }
+    throw error;
+  }
+
+  if (options.debug) {
+    const appPath = getIntegrationPath(appKey);
+    const integrationDir = path.dirname(appPath);
+    const logPath = await writeTestLog(appKey, { request: requestMeta, response: data }, 'test-e2e', integrationDir);
+    logger.log(chalk.gray(`  Debug log: ${logPath}`));
+  }
+
+  return data;
+}
+
+/**
+ * Call E2E API and optionally poll until completed. On throw, caller should log if debug.
+ * @param {Object} opts - Options
+ * @param {string} opts.dataplaneUrl - Dataplane URL
+ * @param {string} opts.datasourceKey - Source key
+ * @param {Object} opts.authConfig - Auth config
+ * @param {Object} opts.body - Request body
+ * @param {boolean} opts.useAsync - Whether to use async + poll
+ * @param {boolean} opts.verbose - Verbose poll progress
+ * @param {number} [opts.pollIntervalMs] - Override poll interval (ms)
+ * @param {number} [opts.pollTimeoutMs] - Override poll timeout (ms)
+ * @returns {Promise<Object>} Final result data
+ */
+/* eslint-disable-next-line max-params -- single opts object; destructuring in body */
+async function executeE2EWithOptionalPoll(opts) {
+  const { dataplaneUrl, datasourceKey, authConfig, body, useAsync, verbose, pollIntervalMs, pollTimeoutMs } = opts;
+  const response = await testDatasourceE2E(dataplaneUrl, datasourceKey, authConfig, body, {
+    asyncRun: useAsync
+  });
+  let data = response.data || response;
+  if (useAsync && data !== null && data !== undefined && data.testRunId) {
+    data = await pollE2ETestRun(
+      dataplaneUrl,
+      datasourceKey,
+      data.testRunId,
+      authConfig,
+      {
+        intervalMs: pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS,
+        timeoutMs: pollTimeoutMs ?? DEFAULT_POLL_TIMEOUT_MS,
+        verbose
+      }
+    );
+  }
+  return data;
+}
+
+module.exports = {
+  runDatasourceTestE2E,
+  resolveAppKey
+};

package/lib/datasource/test-integration.js

@@ -0,0 +1,154 @@
+/**
+ * Datasource integration test - run config test for one datasource via pipeline
+ * @fileoverview Datasource integration test logic
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+/* eslint-disable max-lines-per-function,max-statements,complexity -- Load config, resolve datasource, call pipeline test */
+
+const path = require('path');
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const { getIntegrationPath, resolveIntegrationAppKeyFromCwd } = require('../utils/paths');
+const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
+const { loadConfigFile } = require('../utils/config-format');
+const { setupIntegrationTestAuth } = require('../external-system/test-auth');
+const { getConfig } = require('../core/config');
+const { testDatasourceViaPipeline } = require('../api/pipeline.api');
+const { writeTestLog } = require('../utils/test-log-writer');
+const testHelpers = require('../utils/external-system-test-helpers');
+const fs = require('fs').promises;
+
+/**
+ * Resolve systemKey and appKey for datasource test-integration
+ * @param {string} [appKey] - Explicit app key from --app
+ * @returns {Promise<{appKey: string, systemKey: string}>}
+ */
+async function resolveSystemKey(appKey) {
+  let resolvedAppKey = appKey;
+  if (!resolvedAppKey) {
+    resolvedAppKey = resolveIntegrationAppKeyFromCwd();
+  }
+  if (!resolvedAppKey) {
+    throw new Error(
+      'Could not determine app context. Use --app <appKey> or run from integration/<appKey>/ directory.'
+    );
+  }
+  const appPath = getIntegrationPath(resolvedAppKey);
+  const configPath = resolveApplicationConfigPath(appPath);
+  const config = loadConfigFile(configPath);
+  if (!config.externalIntegration || !config.externalIntegration.systems || config.externalIntegration.systems.length === 0) {
+    throw new Error(`No externalIntegration.systems found in ${configPath}`);
+  }
+  const systemFile = config.externalIntegration.systems[0];
+  const systemPath = path.isAbsolute(systemFile)
+    ? systemFile
+    : path.join(appPath, systemFile);
+  const systemContent = await fs.readFile(systemPath, 'utf8');
+  const yaml = require('js-yaml');
+  const systemConfig = yaml.load(systemContent);
+  const systemKey = systemConfig?.key || path.basename(systemFile, '-system.yaml').replace('-system', '');
+  return { appKey: resolvedAppKey, systemKey };
+}
+
+/**
+ * Run integration test for one datasource
+ * @async
+ * @param {string} datasourceKey - Datasource key
+ * @param {Object} options - Options
+ * @param {string} [options.app] - App key (or resolve from cwd)
+ * @param {string} [options.payload] - Path to custom payload file
+ * @param {string} [options.environment] - Environment (dev, tst, pro)
+ * @param {boolean} [options.debug] - Include debug, write log file
+ * @param {number} [options.timeout] - Request timeout ms
+ * @returns {Promise<Object>} Test result
+ */
+async function runDatasourceTestIntegration(datasourceKey, options = {}) {
+  if (!datasourceKey || typeof datasourceKey !== 'string') {
+    throw new Error('Datasource key is required');
+  }
+  const { appKey, systemKey } = await resolveSystemKey(options.app);
+  const appPath = getIntegrationPath(appKey);
+  const config = loadConfigFile(resolveApplicationConfigPath(appPath));
+  const schemaBasePath = config.externalIntegration?.schemaBasePath || './';
+  const datasourceFiles = config.externalIntegration?.dataSources || [];
+  const datasourceFile = datasourceFiles.find(f => {
+    const base = path.basename(f, path.extname(f));
+    return base === datasourceKey || base.includes(datasourceKey);
+  });
+  if (!datasourceFile) {
+    throw new Error(`Datasource '${datasourceKey}' not found in application config`);
+  }
+  const datasourcePath = path.isAbsolute(schemaBasePath)
+    ? path.join(schemaBasePath, datasourceFile)
+    : path.join(appPath, schemaBasePath, datasourceFile);
+  const datasourceContent = await fs.readFile(datasourcePath, 'utf8');
+  const datasource = JSON.parse(datasourceContent);
+  if (datasource.key !== datasourceKey) {
+    throw new Error(`Datasource key mismatch: file has '${datasource.key}', expected '${datasourceKey}'`);
+  }
+
+  const configObj = await getConfig();
+  const { authConfig, dataplaneUrl } = await setupIntegrationTestAuth(appKey, options, configObj);
+  const customPayload = await testHelpers.loadCustomPayload(options.payload);
+  const payloadTemplate = testHelpers.determinePayloadTemplate(datasource, datasourceKey, customPayload);
+  if (!payloadTemplate) {
+    throw new Error(`No test payload found for datasource '${datasourceKey}'`);
+  }
+
+  logger.log(chalk.blue(`\n📡 Testing datasource: ${datasourceKey} (system: ${systemKey})`));
+
+  const testData = { payloadTemplate };
+  if (options.debug) {
+    testData.includeDebug = true;
+  }
+  const timeout = parseInt(options.timeout, 10) || 30000;
+
+  let response;
+  try {
+    response = await testDatasourceViaPipeline({
+      dataplaneUrl,
+      systemKey,
+      datasourceKey,
+      authConfig,
+      testData,
+      options: { timeout }
+    });
+  } catch (error) {
+    const result = { key: datasourceKey, success: false, error: error.message };
+    if (options.debug) {
+      await writeTestLog(appKey, { request: { systemKey, datasourceKey }, error: error.message }, 'test-integration');
+    }
+    return result;
+  }
+
+  const data = response.data || response;
+  const success = data.success !== false;
+  const result = {
+    key: datasourceKey,
+    systemKey,
+    success,
+    skipped: false,
+    validationResults: data.validationResults || {},
+    fieldMappingResults: data.fieldMappingResults || {},
+    endpointTestResults: data.endpointTestResults || {}
+  };
+  if (data.error) {
+    result.error = data.error;
+  }
+
+  if (options.debug) {
+    const logPath = await writeTestLog(appKey, {
+      request: { systemKey, datasourceKey, includeDebug: true },
+      response: data
+    }, 'test-integration');
+    logger.log(chalk.gray(`  Debug log: ${logPath}`));
+  }
+
+  return result;
+}
+
+module.exports = {
+  runDatasourceTestIntegration,
+  resolveSystemKey
+};

package/lib/deployment/deployer.js

@@ -193,14 +193,16 @@ function buildDeploymentData(validateToken, authConfig, options) {
  * @throws {Error} If deployment failed
  */
 function handleDeploymentResponse(response) {
-  if (response.success) {
+  const bodyFailed = response.data?.success === false;
+  if (response.success && !bodyFailed) {
     return response.data.data || response.data;
   }
 
-  // Handle deployment errors
-  if (response.status >= 400) {
-    const error = new Error(`Deployment request failed: ${response.formattedError || response.error || 'Unknown error'}`);
-    error.status = response.status;
+  // Handle deployment errors (HTTP error or 200 with success: false in body)
+  if (!response.success || bodyFailed || response.status >= 400) {
+    const msg = response.formattedError || response.error || response.data?.formattedError || response.data?.error || 'Unknown error';
+    const error = new Error(`Deployment request failed: ${msg}`);
+    error.status = response.status || (bodyFailed ? 400 : undefined);
     error.data = response.data;
     throw error;
   }