@aifabrix/builder 2.40.2 → 2.42.0

package/lib/datasource/test-e2e.js

@@ -0,0 +1,219 @@
+/**
+ * Datasource E2E test - run full E2E test via dataplane external API
+ * @fileoverview Datasource E2E test logic (config, credential, sync, data, CIP)
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+/* eslint-disable max-statements -- Auth setup, API call, polling, debug log */
+
+const path = require('path');
+const fs = require('fs').promises;
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const { getIntegrationPath, resolveIntegrationAppKeyFromCwd } = require('../utils/paths');
+const { resolveDataplaneUrl } = require('../utils/dataplane-resolver');
+const { resolveControllerUrl } = require('../utils/controller-url');
+const { getDeviceOnlyAuth } = require('../utils/token-manager');
+const { testDatasourceE2E, getE2ETestRun } = require('../api/external-test.api');
+const { writeTestLog } = require('../utils/test-log-writer');
+
+const DEFAULT_POLL_INTERVAL_MS = 2500;
+const DEFAULT_POLL_TIMEOUT_MS = 15 * 60 * 1000;
+
+/**
+ * Resolve appKey for datasource test-e2e
+ * @param {string} [appKey] - Explicit app key from --app
+ * @returns {string}
+ */
+function resolveAppKey(appKey) {
+  if (appKey) return appKey;
+  const fromCwd = resolveIntegrationAppKeyFromCwd();
+  if (fromCwd) return fromCwd;
+  throw new Error(
+    'Could not determine app context. Use --app <appKey> or run from integration/<appKey>/ directory.'
+  );
+}
+
+/**
+ * Resolve primaryKeyValue for request body: string as-is, or read and parse JSON from @path
+ * @param {string} [value] - Literal value or path prefixed with @ (e.g. @pk.json)
+ * @returns {Promise<string|Object|null>} Resolved value for body.primaryKeyValue, or null if absent
+ */
+async function resolvePrimaryKeyValue(value) {
+  if (value === null || value === undefined || value === '') return null;
+  const str = String(value).trim();
+  if (str.startsWith('@')) {
+    const filePath = path.resolve(str.slice(1).trim());
+    const content = await fs.readFile(filePath, 'utf8');
+    return JSON.parse(content);
+  }
+  return str;
+}
+
+/**
+ * Build E2E request body from options
+ * @param {Object} options - Command options
+ * @returns {Promise<Object>} Request body
+ */
+async function buildE2EBody(options) {
+  const body = {};
+  if (options.debug) body.includeDebug = true;
+  if (options.testCrud === true) body.testCrud = true;
+  if (options.recordId !== undefined && options.recordId !== null && options.recordId !== '') body.recordId = String(options.recordId);
+  if (options.cleanup === false) body.cleanup = false;
+  else if (options.cleanup === true) body.cleanup = true;
+  const pk = await resolvePrimaryKeyValue(options.primaryKeyValue);
+  if (pk !== null && pk !== undefined) body.primaryKeyValue = pk;
+  return body;
+}
+
+/**
+ * Poll E2E test run until completed or failed
+ * @param {string} dataplaneUrl - Dataplane URL
+ * @param {string} sourceIdOrKey - Source ID or key
+ * @param {string} testRunId - Test run ID
+ * @param {Object} authConfig - Auth config
+ * @param {Object} opts - Poll options
+ * @param {number} [opts.intervalMs] - Poll interval (ms)
+ * @param {number} [opts.timeoutMs] - Max wait (ms)
+ * @param {boolean} [opts.verbose] - Log each poll
+ * @returns {Promise<Object>} Final poll result (status completed or failed)
+ */
+async function pollE2ETestRun(dataplaneUrl, sourceIdOrKey, testRunId, authConfig, opts = {}) {
+  const intervalMs = opts.intervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_POLL_TIMEOUT_MS;
+  const verbose = opts.verbose === true;
+  const deadline = Date.now() + timeoutMs;
+  let last;
+  while (Date.now() < deadline) {
+    last = await getE2ETestRun(dataplaneUrl, sourceIdOrKey, testRunId, authConfig);
+    if (last.status === 'completed' || last.status === 'failed') {
+      return last;
+    }
+    if (verbose) {
+      const steps = last.completedActions || [];
+      logger.log(chalk.gray(`  Polling… status: ${last.status}, ${steps.length} step(s) completed`));
+    }
+    await new Promise(r => setTimeout(r, intervalMs));
+  }
+  throw new Error(
+    `E2E test run did not complete within ${timeoutMs / 1000}s (run ID: ${testRunId})`
+  );
+}
+
+/**
+ * Run E2E test for one datasource (Bearer token or API key required; no client credentials).
+ * Default: async start + polling until completed/failed. Use options.async === false for sync.
+ *
+ * @async
+ * @param {string} datasourceKey - Datasource key (used as sourceIdOrKey)
+ * @param {Object} options - Options
+ * @param {string} [options.app] - App key (or resolve from cwd)
+ * @param {string} [options.environment] - Environment (dev, tst, pro)
+ * @param {boolean} [options.debug] - Include debug, write log file
+ * @param {boolean} [options.verbose] - Verbose output (e.g. poll progress)
+ * @param {boolean} [options.async] - If false, use sync mode (no polling). Default true.
+ * @param {boolean} [options.testCrud] - Set body testCrud true
+ * @param {string} [options.recordId] - Set body recordId
+ * @param {boolean} [options.cleanup] - Set body cleanup (default true)
+ * @param {string} [options.primaryKeyValue] - Set body primaryKeyValue (string or @path to JSON)
+ * @param {number} [options.pollIntervalMs] - Poll interval in ms (default 2500)
+ * @param {number} [options.pollTimeoutMs] - Poll timeout in ms (default 15 min)
+ * @returns {Promise<Object>} E2E test result (steps, success, error, etc.)
+ */
+async function runDatasourceTestE2E(datasourceKey, options = {}) {
+  if (!datasourceKey || typeof datasourceKey !== 'string') {
+    throw new Error('Datasource key is required');
+  }
+  const appKey = resolveAppKey(options.app);
+  const controllerUrl = await resolveControllerUrl();
+  const { resolveEnvironment } = require('../core/config');
+  const environment = options.environment || await resolveEnvironment();
+  const authConfig = await getDeviceOnlyAuth(controllerUrl);
+  const dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig);
+
+  logger.log(chalk.blue(`\n🧪 Running E2E test for datasource: ${datasourceKey}`));
+
+  const body = await buildE2EBody(options);
+  const useAsync = options.async !== false;
+  const requestMeta = {
+    sourceIdOrKey: datasourceKey,
+    includeDebug: options.debug,
+    testCrud: options.testCrud,
+    recordId: options.recordId,
+    cleanup: options.cleanup,
+    primaryKeyValue: options.primaryKeyValue !== undefined && options.primaryKeyValue !== null
+  };
+
+  const execOpts = {
+    dataplaneUrl,
+    datasourceKey,
+    authConfig,
+    body,
+    useAsync,
+    verbose: options.verbose,
+    pollIntervalMs: options.pollIntervalMs,
+    pollTimeoutMs: options.pollTimeoutMs
+  };
+  let data;
+  try {
+    data = await executeE2EWithOptionalPoll(execOpts);
+  } catch (error) {
+    if (options.debug) {
+      const appPath = getIntegrationPath(appKey);
+      const integrationDir = path.dirname(appPath);
+      await writeTestLog(appKey, { request: requestMeta, error: error.message }, 'test-e2e', integrationDir);
+    }
+    throw error;
+  }
+
+  if (options.debug) {
+    const appPath = getIntegrationPath(appKey);
+    const integrationDir = path.dirname(appPath);
+    const logPath = await writeTestLog(appKey, { request: requestMeta, response: data }, 'test-e2e', integrationDir);
+    logger.log(chalk.gray(`  Debug log: ${logPath}`));
+  }
+
+  return data;
+}
+
+/**
+ * Call E2E API and optionally poll until completed. On throw, caller should log if debug.
+ * @param {Object} opts - Options
+ * @param {string} opts.dataplaneUrl - Dataplane URL
+ * @param {string} opts.datasourceKey - Source key
+ * @param {Object} opts.authConfig - Auth config
+ * @param {Object} opts.body - Request body
+ * @param {boolean} opts.useAsync - Whether to use async + poll
+ * @param {boolean} opts.verbose - Verbose poll progress
+ * @param {number} [opts.pollIntervalMs] - Override poll interval (ms)
+ * @param {number} [opts.pollTimeoutMs] - Override poll timeout (ms)
+ * @returns {Promise<Object>} Final result data
+ */
+/* eslint-disable-next-line max-params -- single opts object; destructuring in body */
+async function executeE2EWithOptionalPoll(opts) {
+  const { dataplaneUrl, datasourceKey, authConfig, body, useAsync, verbose, pollIntervalMs, pollTimeoutMs } = opts;
+  const response = await testDatasourceE2E(dataplaneUrl, datasourceKey, authConfig, body, {
+    asyncRun: useAsync
+  });
+  let data = response.data || response;
+  if (useAsync && data !== null && data !== undefined && data.testRunId) {
+    data = await pollE2ETestRun(
+      dataplaneUrl,
+      datasourceKey,
+      data.testRunId,
+      authConfig,
+      {
+        intervalMs: pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS,
+        timeoutMs: pollTimeoutMs ?? DEFAULT_POLL_TIMEOUT_MS,
+        verbose
+      }
+    );
+  }
+  return data;
+}
+
+module.exports = {
+  runDatasourceTestE2E,
+  resolveAppKey
+};

package/lib/datasource/test-integration.js

@@ -0,0 +1,154 @@
+/**
+ * Datasource integration test - run config test for one datasource via pipeline
+ * @fileoverview Datasource integration test logic
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+/* eslint-disable max-lines-per-function,max-statements,complexity -- Load config, resolve datasource, call pipeline test */
+
+const path = require('path');
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const { getIntegrationPath, resolveIntegrationAppKeyFromCwd } = require('../utils/paths');
+const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
+const { loadConfigFile } = require('../utils/config-format');
+const { setupIntegrationTestAuth } = require('../external-system/test-auth');
+const { getConfig } = require('../core/config');
+const { testDatasourceViaPipeline } = require('../api/pipeline.api');
+const { writeTestLog } = require('../utils/test-log-writer');
+const testHelpers = require('../utils/external-system-test-helpers');
+const fs = require('fs').promises;
+
+/**
+ * Resolve systemKey and appKey for datasource test-integration
+ * @param {string} [appKey] - Explicit app key from --app
+ * @returns {Promise<{appKey: string, systemKey: string}>}
+ */
+async function resolveSystemKey(appKey) {
+  let resolvedAppKey = appKey;
+  if (!resolvedAppKey) {
+    resolvedAppKey = resolveIntegrationAppKeyFromCwd();
+  }
+  if (!resolvedAppKey) {
+    throw new Error(
+      'Could not determine app context. Use --app <appKey> or run from integration/<appKey>/ directory.'
+    );
+  }
+  const appPath = getIntegrationPath(resolvedAppKey);
+  const configPath = resolveApplicationConfigPath(appPath);
+  const config = loadConfigFile(configPath);
+  if (!config.externalIntegration || !config.externalIntegration.systems || config.externalIntegration.systems.length === 0) {
+    throw new Error(`No externalIntegration.systems found in ${configPath}`);
+  }
+  const systemFile = config.externalIntegration.systems[0];
+  const systemPath = path.isAbsolute(systemFile)
+    ? systemFile
+    : path.join(appPath, systemFile);
+  const systemContent = await fs.readFile(systemPath, 'utf8');
+  const yaml = require('js-yaml');
+  const systemConfig = yaml.load(systemContent);
+  const systemKey = systemConfig?.key || path.basename(systemFile, '-system.yaml').replace('-system', '');
+  return { appKey: resolvedAppKey, systemKey };
+}
+
+/**
+ * Run integration test for one datasource
+ * @async
+ * @param {string} datasourceKey - Datasource key
+ * @param {Object} options - Options
+ * @param {string} [options.app] - App key (or resolve from cwd)
+ * @param {string} [options.payload] - Path to custom payload file
+ * @param {string} [options.environment] - Environment (dev, tst, pro)
+ * @param {boolean} [options.debug] - Include debug, write log file
+ * @param {number} [options.timeout] - Request timeout ms
+ * @returns {Promise<Object>} Test result
+ */
+async function runDatasourceTestIntegration(datasourceKey, options = {}) {
+  if (!datasourceKey || typeof datasourceKey !== 'string') {
+    throw new Error('Datasource key is required');
+  }
+  const { appKey, systemKey } = await resolveSystemKey(options.app);
+  const appPath = getIntegrationPath(appKey);
+  const config = loadConfigFile(resolveApplicationConfigPath(appPath));
+  const schemaBasePath = config.externalIntegration?.schemaBasePath || './';
+  const datasourceFiles = config.externalIntegration?.dataSources || [];
+  const datasourceFile = datasourceFiles.find(f => {
+    const base = path.basename(f, path.extname(f));
+    return base === datasourceKey || base.includes(datasourceKey);
+  });
+  if (!datasourceFile) {
+    throw new Error(`Datasource '${datasourceKey}' not found in application config`);
+  }
+  const datasourcePath = path.isAbsolute(schemaBasePath)
+    ? path.join(schemaBasePath, datasourceFile)
+    : path.join(appPath, schemaBasePath, datasourceFile);
+  const datasourceContent = await fs.readFile(datasourcePath, 'utf8');
+  const datasource = JSON.parse(datasourceContent);
+  if (datasource.key !== datasourceKey) {
+    throw new Error(`Datasource key mismatch: file has '${datasource.key}', expected '${datasourceKey}'`);
+  }
+
+  const configObj = await getConfig();
+  const { authConfig, dataplaneUrl } = await setupIntegrationTestAuth(appKey, options, configObj);
+  const customPayload = await testHelpers.loadCustomPayload(options.payload);
+  const payloadTemplate = testHelpers.determinePayloadTemplate(datasource, datasourceKey, customPayload);
+  if (!payloadTemplate) {
+    throw new Error(`No test payload found for datasource '${datasourceKey}'`);
+  }
+
+  logger.log(chalk.blue(`\n📡 Testing datasource: ${datasourceKey} (system: ${systemKey})`));
+
+  const testData = { payloadTemplate };
+  if (options.debug) {
+    testData.includeDebug = true;
+  }
+  const timeout = parseInt(options.timeout, 10) || 30000;
+
+  let response;
+  try {
+    response = await testDatasourceViaPipeline({
+      dataplaneUrl,
+      systemKey,
+      datasourceKey,
+      authConfig,
+      testData,
+      options: { timeout }
+    });
+  } catch (error) {
+    const result = { key: datasourceKey, success: false, error: error.message };
+    if (options.debug) {
+      await writeTestLog(appKey, { request: { systemKey, datasourceKey }, error: error.message }, 'test-integration');
+    }
+    return result;
+  }
+
+  const data = response.data || response;
+  const success = data.success !== false;
+  const result = {
+    key: datasourceKey,
+    systemKey,
+    success,
+    skipped: false,
+    validationResults: data.validationResults || {},
+    fieldMappingResults: data.fieldMappingResults || {},
+    endpointTestResults: data.endpointTestResults || {}
+  };
+  if (data.error) {
+    result.error = data.error;
+  }
+
+  if (options.debug) {
+    const logPath = await writeTestLog(appKey, {
+      request: { systemKey, datasourceKey, includeDebug: true },
+      response: data
+    }, 'test-integration');
+    logger.log(chalk.gray(`  Debug log: ${logPath}`));
+  }
+
+  return result;
+}
+
+module.exports = {
+  runDatasourceTestIntegration,
+  resolveSystemKey
+};

package/lib/datasource/validate.js

@@ -11,6 +11,7 @@
 const fs = require('fs');
 const { loadExternalDataSourceSchema } = require('../utils/schema-loader');
 const { formatValidationErrors } = require('../utils/error-formatter');
+const { validateFieldReferences } = require('./field-reference-validator');
 
 /**
  * Validates a datasource file against external-datasource schema
@@ -48,11 +49,28 @@ async function validateDatasourceFile(filePath) {
   }
 
   const validate = loadExternalDataSourceSchema();
-  const valid = validate(parsed);
+  const schemaValid = validate(parsed);
+
+  if (!schemaValid) {
+    return {
+      valid: false,
+      errors: formatValidationErrors(validate.errors),
+      warnings: []
+    };
+  }
+
+  const fieldRefErrors = validateFieldReferences(parsed);
+  if (fieldRefErrors.length > 0) {
+    return {
+      valid: false,
+      errors: fieldRefErrors,
+      warnings: []
+    };
+  }
 
   return {
-    valid,
-    errors: valid ? [] : formatValidationErrors(validate.errors),
+    valid: true,
+    errors: [],
     warnings: []
   };
 }

package/lib/deployment/deployer.js

@@ -193,14 +193,16 @@ function buildDeploymentData(validateToken, authConfig, options) {
  * @throws {Error} If deployment failed
  */
 function handleDeploymentResponse(response) {
-  if (response.success) {
+  const bodyFailed = response.data?.success === false;
+  if (response.success && !bodyFailed) {
     return response.data.data || response.data;
   }
 
-  // Handle deployment errors
-  if (response.status >= 400) {
-    const error = new Error(`Deployment request failed: ${response.formattedError || response.error || 'Unknown error'}`);
-    error.status = response.status;
+  // Handle deployment errors (HTTP error or 200 with success: false in body)
+  if (!response.success || bodyFailed || response.status >= 400) {
+    const msg = response.formattedError || response.error || response.data?.formattedError || response.data?.error || 'Unknown error';
+    const error = new Error(`Deployment request failed: ${msg}`);
+    error.status = response.status || (bodyFailed ? 400 : undefined);
     error.data = response.data;
     throw error;
   }

package/lib/deployment/environment-config.js

@@ -0,0 +1,137 @@
+/**
+ * Environment deployment config and preset helpers.
+ * Parses/validates JSON config and builds preset-based config.
+ *
+ * @fileoverview Environment config for AI Fabrix Builder
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fs = require('fs');
+const path = require('path');
+const Ajv = require('ajv');
+const { formatValidationErrors } = require('../utils/error-formatter');
+const environmentDeployRequestSchema = require('../schema/environment-deploy-request.schema.json');
+
+/** Allowed preset values for --preset (case-insensitive); API expects lowercase s, m, l, xl */
+const PRESET_VALUES = ['s', 'm', 'l', 'xl'];
+const DEFAULT_PRESET = 's';
+const DEFAULT_SERVICE_NAME = 'aifabrix';
+const DEFAULT_LOCATION = 'swedencentral';
+
+/**
+ * Normalizes and validates preset from CLI (s, m, l, xl; case-insensitive)
+ * @param {string} [preset] - User-provided preset
+ * @returns {string} Normalized preset (lowercase)
+ * @throws {Error} If preset is not one of s, m, l, xl
+ */
+function normalizePreset(preset) {
+  const raw = (preset === null || preset === undefined || preset === '') ? DEFAULT_PRESET : String(preset).trim().toLowerCase();
+  if (!PRESET_VALUES.includes(raw)) {
+    throw new Error(
+      `Invalid preset "${preset}". Use one of: ${PRESET_VALUES.join(', ')}.\n` +
+      'Example: aifabrix env deploy dev --preset s'
+    );
+  }
+  return raw;
+}
+
+/**
+ * Builds environmentConfig from env key and preset (no config file)
+ * @param {string} validatedEnvKey - Validated environment key
+ * @param {string} preset - Normalized preset (s, m, l, xl)
+ * @returns {Object} { environmentConfig, dryRun: false }
+ */
+function buildEnvironmentConfigFromPreset(validatedEnvKey, preset) {
+  return {
+    environmentConfig: {
+      key: validatedEnvKey,
+      environment: validatedEnvKey,
+      preset,
+      serviceName: DEFAULT_SERVICE_NAME,
+      location: DEFAULT_LOCATION
+    },
+    dryRun: false
+  };
+}
+
+/** Reads and parses config file; throws if missing, unreadable, or invalid structure. */
+function parseEnvironmentConfigFile(resolvedPath) {
+  let raw;
+  try {
+    raw = fs.readFileSync(resolvedPath, 'utf8');
+  } catch (e) {
+    throw new Error(`Cannot read config file: ${resolvedPath}. ${e.message}`);
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (e) {
+    throw new Error(
+      `Invalid JSON in config file: ${resolvedPath}\n${e.message}\n` +
+      'Expected format: { "environmentConfig": { "key", "environment", "preset", "serviceName", "location" }, "dryRun": false }'
+    );
+  }
+  if (parsed === null || typeof parsed !== 'object') {
+    throw new Error(
+      `Config file must be a JSON object with "environmentConfig". File: ${resolvedPath}\n` +
+      'Example: { "environmentConfig": { "key": "dev", "environment": "dev", "preset": "s", "serviceName": "aifabrix", "location": "swedencentral" }, "dryRun": false }'
+    );
+  }
+  if (parsed.environmentConfig === undefined) {
+    throw new Error(
+      `Config file must contain "environmentConfig" (object). File: ${resolvedPath}\n` +
+      'Example: { "environmentConfig": { "key": "dev", "environment": "dev", "preset": "s", "serviceName": "aifabrix", "location": "swedencentral" } }'
+    );
+  }
+  if (typeof parsed.environmentConfig !== 'object' || parsed.environmentConfig === null) {
+    throw new Error(`"environmentConfig" must be an object. File: ${resolvedPath}`);
+  }
+  return parsed;
+}
+
+/**
+ * Validates parsed config against schema and returns deploy request.
+ * @param {Object} parsed - Parsed config object
+ * @param {string} resolvedPath - Path for error messages
+ * @returns {Object} { environmentConfig, dryRun? }
+ */
+function validateEnvironmentDeployParsed(parsed, resolvedPath) {
+  const ajv = new Ajv({ allErrors: true, strict: false });
+  const validate = ajv.compile(environmentDeployRequestSchema);
+  if (!validate(parsed)) {
+    const messages = formatValidationErrors(validate.errors);
+    throw new Error(
+      `Environment config validation failed (${resolvedPath}):\n  • ${messages.join('\n  • ')}\n` +
+      'Fix the config file and run the command again. See templates/infra/environment-dev.json for a valid example.'
+    );
+  }
+  return {
+    environmentConfig: parsed.environmentConfig,
+    dryRun: Boolean(parsed.dryRun)
+  };
+}
+
+/**
+ * Loads and validates environment deploy config from a JSON file
+ * @param {string} configPath - Absolute or relative path to config JSON
+ * @returns {Object} Valid deploy request { environmentConfig, dryRun? }
+ * @throws {Error} If file missing, invalid JSON, or validation fails
+ */
+function loadAndValidateEnvironmentDeployConfig(configPath) {
+  const resolvedPath = path.isAbsolute(configPath) ? configPath : path.resolve(process.cwd(), configPath);
+  if (!fs.existsSync(resolvedPath)) {
+    throw new Error(
+      `Environment config file not found: ${resolvedPath}\n` +
+      'Use --config <file> with a JSON file containing "environmentConfig" (e.g. templates/infra/environment-dev.json).'
+    );
+  }
+  const parsed = parseEnvironmentConfigFile(resolvedPath);
+  return validateEnvironmentDeployParsed(parsed, resolvedPath);
+}
+
+module.exports = {
+  normalizePreset,
+  buildEnvironmentConfigFromPreset,
+  loadAndValidateEnvironmentDeployConfig
+};