@aifabrix/builder 2.42.1 → 2.44.0

package/lib/datasource/test-integration.js

@@ -1,154 +1,166 @@
 /**
- * Datasource integration test - run config test for one datasource via pipeline
+ * Datasource integration test — unified dataplane validation (runType=integration).
  * @fileoverview Datasource integration test logic
  * @author AI Fabrix Team
  * @version 2.0.0
  */
-/* eslint-disable max-lines-per-function,max-statements,complexity -- Load config, resolve datasource, call pipeline test */
 
-const path = require('path');
 const chalk = require('chalk');
 const logger = require('../utils/logger');
-const { getIntegrationPath, resolveIntegrationAppKeyFromCwd } = require('../utils/paths');
-const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
-const { loadConfigFile } = require('../utils/config-format');
-const { setupIntegrationTestAuth } = require('../external-system/test-auth');
-const { getConfig } = require('../core/config');
-const { testDatasourceViaPipeline } = require('../api/pipeline.api');
+const { resolveAppKeyForDatasource } = require('./resolve-app');
+const { infoLine } = require('../utils/cli-test-layout-chalk');
+const {
+  getSystemKeyFromAppKey,
+  findDatasourceFileByKey
+} = require('./integration-context');
+const { runUnifiedDatasourceValidation } = require('./unified-validation-run');
+const { integrationResultFromEnvelope } = require('../utils/datasource-test-run-legacy-adapter');
 const { writeTestLog } = require('../utils/test-log-writer');
-const testHelpers = require('../utils/external-system-test-helpers');
-const fs = require('fs').promises;
 
 /**
- * Resolve systemKey and appKey for datasource test-integration
- * @param {string} [appKey] - Explicit app key from --app
- * @returns {Promise<{appKey: string, systemKey: string}>}
+ * @param {string} appKey - Integration folder name (same as --app; system key in publish flows)
+ * @returns {Promise<string>} systemKey
  */
-async function resolveSystemKey(appKey) {
-  let resolvedAppKey = appKey;
-  if (!resolvedAppKey) {
-    resolvedAppKey = resolveIntegrationAppKeyFromCwd();
+async function getSystemKeyFromAppKeyExport(appKey) {
+  return getSystemKeyFromAppKey(appKey);
+}
+
+function legacyFailureShell(datasourceKey, systemKey, error, datasourceTestRun, runMeta) {
+  return {
+    key: datasourceKey,
+    systemKey,
+    success: false,
+    skipped: false,
+    validationResults: {},
+    fieldMappingResults: {},
+    endpointTestResults: {},
+    error,
+    datasourceTestRun,
+    runMeta
+  };
+}
+
+/**
+ * @returns {{ body: Object, apiErrMsg?: string }|null}
+ */
+function integrationEarlyExitBody(datasourceKey, systemKey, unifiedResult, runMeta) {
+  if (unifiedResult.apiError) {
+    const errMsg =
+      unifiedResult.apiError.formattedError ||
+      unifiedResult.apiError.error ||
+      'Request failed';
+    return {
+      body: legacyFailureShell(datasourceKey, systemKey, errMsg, null, runMeta),
+      apiErrMsg: errMsg
+    };
   }
-  if (!resolvedAppKey) {
-    throw new Error(
-      'Could not determine app context. Use --app <appKey> or run from integration/<appKey>/ directory.'
-    );
+  if (unifiedResult.pollTimedOut) {
+    return {
+      body: legacyFailureShell(
+        datasourceKey,
+        systemKey,
+        'Report incomplete: timeout',
+        unifiedResult.envelope,
+        runMeta
+      )
+    };
   }
-  const appPath = getIntegrationPath(resolvedAppKey);
-  const configPath = resolveApplicationConfigPath(appPath);
-  const config = loadConfigFile(configPath);
-  if (!config.externalIntegration || !config.externalIntegration.systems || config.externalIntegration.systems.length === 0) {
-    throw new Error(`No externalIntegration.systems found in ${configPath}`);
+  if (unifiedResult.incompleteNoAsync) {
+    return {
+      body: legacyFailureShell(
+        datasourceKey,
+        systemKey,
+        'Report incomplete (async required)',
+        unifiedResult.envelope,
+        runMeta
+      )
+    };
   }
-  const systemFile = config.externalIntegration.systems[0];
-  const systemPath = path.isAbsolute(systemFile)
-    ? systemFile
-    : path.join(appPath, systemFile);
-  const systemContent = await fs.readFile(systemPath, 'utf8');
-  const yaml = require('js-yaml');
-  const systemConfig = yaml.load(systemContent);
-  const systemKey = systemConfig?.key || path.basename(systemFile, '-system.yaml').replace('-system', '');
-  return { appKey: resolvedAppKey, systemKey };
+  return null;
 }
 
 /**
- * Run integration test for one datasource
+ * Run integration test for one datasource (unified validation API).
  * @async
  * @param {string} datasourceKey - Datasource key
  * @param {Object} options - Options
  * @param {string} [options.app] - App key (or resolve from cwd)
  * @param {string} [options.payload] - Path to custom payload file
  * @param {string} [options.environment] - Environment (dev, tst, pro)
- * @param {boolean} [options.debug] - Include debug, write log file
- * @param {number} [options.timeout] - Request timeout ms
- * @returns {Promise<Object>} Test result
+ * @param {boolean} [options.verbose] - explain=true on request
+ * @param {boolean|string} [options.debug] - Truthy enables includeDebug and log file; string `summary`|`full`|`raw` selects TTY appendix (CLI)
+ * @param {number|string} [options.timeout] - Aggregate timeout ms
+ * @param {boolean} [options.sync] - Publish local datasource JSON before validation when true
+ * @returns {Promise<Object>} Legacy-shaped result + datasourceTestRun / runMeta when present
  */
+function logIntegrationDatasourceBanner(datasourceKey, systemKey, verbose) {
+  if (!verbose) return;
+  logger.log('');
+  logger.log(infoLine(`📡 Testing datasource: ${datasourceKey} (system: ${systemKey})`));
+}
+
 async function runDatasourceTestIntegration(datasourceKey, options = {}) {
   if (!datasourceKey || typeof datasourceKey !== 'string') {
     throw new Error('Datasource key is required');
   }
-  const { appKey, systemKey } = await resolveSystemKey(options.app);
-  const appPath = getIntegrationPath(appKey);
-  const config = loadConfigFile(resolveApplicationConfigPath(appPath));
-  const schemaBasePath = config.externalIntegration?.schemaBasePath || './';
-  const datasourceFiles = config.externalIntegration?.dataSources || [];
-  const datasourceFile = datasourceFiles.find(f => {
-    const base = path.basename(f, path.extname(f));
-    return base === datasourceKey || base.includes(datasourceKey);
-  });
-  if (!datasourceFile) {
-    throw new Error(`Datasource '${datasourceKey}' not found in application config`);
-  }
-  const datasourcePath = path.isAbsolute(schemaBasePath)
-    ? path.join(schemaBasePath, datasourceFile)
-    : path.join(appPath, schemaBasePath, datasourceFile);
-  const datasourceContent = await fs.readFile(datasourcePath, 'utf8');
-  const datasource = JSON.parse(datasourceContent);
-  if (datasource.key !== datasourceKey) {
-    throw new Error(`Datasource key mismatch: file has '${datasource.key}', expected '${datasourceKey}'`);
-  }
+  const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
+  const systemKey = await getSystemKeyFromAppKey(appKey);
 
-  const configObj = await getConfig();
-  const { authConfig, dataplaneUrl } = await setupIntegrationTestAuth(appKey, options, configObj);
-  const customPayload = await testHelpers.loadCustomPayload(options.payload);
-  const payloadTemplate = testHelpers.determinePayloadTemplate(datasource, datasourceKey, customPayload);
-  if (!payloadTemplate) {
-    throw new Error(`No test payload found for datasource '${datasourceKey}'`);
-  }
+  logIntegrationDatasourceBanner(datasourceKey, systemKey, options.verbose);
 
-  logger.log(chalk.blue(`\n📡 Testing datasource: ${datasourceKey} (system: ${systemKey})`));
+  const unifiedResult = await runUnifiedDatasourceValidation(datasourceKey, {
+    app: options.app,
+    environment: options.environment,
+    runType: 'integration',
+    payload: options.payload,
+    debug: options.debug,
+    verbose: options.verbose,
+    timeout: options.timeout,
+    async: true,
+    noAsync: false,
+    sync: options.sync === true
+  });
 
-  const testData = { payloadTemplate };
-  if (options.debug) {
-    testData.includeDebug = true;
-  }
-  const timeout = parseInt(options.timeout, 10) || 30000;
+  const runMeta = {
+    apiError: unifiedResult.apiError,
+    pollTimedOut: unifiedResult.pollTimedOut,
+    incompleteNoAsync: unifiedResult.incompleteNoAsync
+  };
 
-  let response;
-  try {
-    response = await testDatasourceViaPipeline({
-      dataplaneUrl,
-      systemKey,
-      datasourceKey,
-      authConfig,
-      testData,
-      options: { timeout }
-    });
-  } catch (error) {
-    const result = { key: datasourceKey, success: false, error: error.message };
-    if (options.debug) {
-      await writeTestLog(appKey, { request: { systemKey, datasourceKey }, error: error.message }, 'test-integration');
+  const early = integrationEarlyExitBody(datasourceKey, systemKey, unifiedResult, runMeta);
+  if (early) {
+    if (early.apiErrMsg && options.debug) {
+      await writeTestLog(
+        appKey,
+        { request: { systemKey, datasourceKey }, error: early.apiErrMsg },
+        'test-integration'
+      );
     }
-    return result;
+    return early.body;
   }
 
-  const data = response.data || response;
-  const success = data.success !== false;
-  const result = {
-    key: datasourceKey,
-    systemKey,
-    success,
-    skipped: false,
-    validationResults: data.validationResults || {},
-    fieldMappingResults: data.fieldMappingResults || {},
-    endpointTestResults: data.endpointTestResults || {}
-  };
-  if (data.error) {
-    result.error = data.error;
-  }
+  const legacy = integrationResultFromEnvelope(unifiedResult.envelope, datasourceKey);
+  legacy.systemKey = systemKey;
+  legacy.datasourceTestRun = unifiedResult.envelope;
+  legacy.runMeta = { apiError: null, pollTimedOut: false, incompleteNoAsync: false };
 
-  if (options.debug) {
-    const logPath = await writeTestLog(appKey, {
-      request: { systemKey, datasourceKey, includeDebug: true },
-      response: data
-    }, 'test-integration');
+  if (options.debug && unifiedResult.envelope) {
+    const logPath = await writeTestLog(
+      appKey,
+      {
+        request: { systemKey, datasourceKey, includeDebug: true },
+        response: unifiedResult.envelope
+      },
+      'test-integration'
+    );
     logger.log(chalk.gray(`  Debug log: ${logPath}`));
   }
 
-  return result;
+  return legacy;
 }
 
 module.exports = {
   runDatasourceTestIntegration,
-  resolveSystemKey
+  getSystemKeyFromAppKey: getSystemKeyFromAppKeyExport,
+  findDatasourceFileByKey
 };

package/lib/datasource/unified-validation-run-body.js

@@ -0,0 +1,65 @@
+/**
+ * @fileoverview Build ValidationRunRequest body for unified datasource runs.
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const {
+  buildExternalDataSourceValidationRequest,
+  buildE2eOptionsFromCli,
+  includeDebugForRequest
+} = require('../utils/validation-run-request');
+const testHelpers = require('../utils/external-system-test-helpers');
+
+/**
+ * @param {Object} params
+ * @param {string} params.systemKey
+ * @param {string} params.datasourceKey
+ * @param {'test'|'integration'|'e2e'} params.runType
+ * @param {Object} params.datasource - Loaded datasource config
+ * @param {string} [params.payloadPath]
+ * @param {boolean} params.useAsync
+ * @param {Object} params.options - CLI options (debug, verbose, e2e fields, capabilityKey)
+ * @returns {Promise<import('../api/types/validation-run.types').ValidationRunRequestBody>}
+ */
+async function buildUnifiedValidationBody(params) {
+  const { systemKey, datasourceKey, runType, datasource, payloadPath, useAsync, options } = params;
+
+  let payloadTemplate;
+  if (payloadPath || runType === 'integration') {
+    const customPayload = await testHelpers.loadCustomPayload(payloadPath);
+    payloadTemplate = testHelpers.determinePayloadTemplate(datasource, datasourceKey, customPayload);
+    if (runType === 'integration' && !payloadTemplate) {
+      throw new Error(`No test payload found for datasource '${datasourceKey}'`);
+    }
+  }
+
+  const e2eExtra = options.capabilityKey
+    ? { capabilityKeys: [String(options.capabilityKey).trim()] }
+    : undefined;
+  const e2eOptions =
+    runType === 'e2e'
+      ? buildE2eOptionsFromCli({
+        debug: options.debug,
+        verbose: options.verbose,
+        testCrud: options.testCrud,
+        recordId: options.recordId,
+        cleanup: options.cleanup,
+        primaryKeyValue: options.primaryKeyValue,
+        e2eOptionsExtra: e2eExtra
+      })
+      : undefined;
+
+  return buildExternalDataSourceValidationRequest({
+    systemKey,
+    datasourceKey,
+    runType,
+    payloadTemplate,
+    asyncRun: runType === 'e2e' && useAsync === true,
+    includeDebug: includeDebugForRequest(options.debug),
+    explain: options.verbose === true,
+    e2eOptions: e2eOptions && Object.keys(e2eOptions).length > 0 ? e2eOptions : undefined
+  });
+}
+
+module.exports = { buildUnifiedValidationBody };

package/lib/datasource/unified-validation-run-post.js

@@ -0,0 +1,23 @@
+/**
+ * @fileoverview POST + optional poll for unified validation run (keeps main module small).
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const { postValidationRunAndOptionalPoll } = require('../api/validation-runner');
+
+/**
+ * @param {Object} opts
+ * @param {string} opts.dataplaneUrl
+ * @param {Object} opts.authConfig
+ * @param {Object} opts.body
+ * @param {number} opts.timeoutMs
+ * @param {boolean} opts.useAsync
+ * @param {boolean} opts.noAsync
+ * @returns {Promise<{ envelope: Object|null, apiError: Object|null, pollTimedOut: boolean, incompleteNoAsync: boolean }>}
+ */
+/* eslint-disable max-lines-per-function, max-statements, complexity -- POST + poll orchestration */
+// Re-exported thin wrapper; single implementation lives in lib/api/validation-runner.js
+/* eslint-enable max-lines-per-function, max-statements, complexity */
+
+module.exports = { postValidationRunAndOptionalPoll };

package/lib/datasource/unified-validation-run-resolve.js

@@ -0,0 +1,43 @@
+/**
+ * @fileoverview Resolve datasource file path + parsed config for unified validation.
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const path = require('path');
+const { getIntegrationPath } = require('../utils/paths');
+const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
+const { loadConfigFile } = require('../utils/config-format');
+
+/**
+ * @param {string} appKey
+ * @param {string} datasourceKey
+ * @param {Function} findDatasourceFileByKey - From integration-context
+ * @returns {{ datasource: Object, datasourcePath: string }}
+ */
+function loadDatasourceForApp(appKey, datasourceKey, findDatasourceFileByKey) {
+  const appPath = getIntegrationPath(appKey);
+  const config = loadConfigFile(resolveApplicationConfigPath(appPath));
+  const schemaBasePath = config.externalIntegration?.schemaBasePath || './';
+  const datasourceFiles = config.externalIntegration?.dataSources || [];
+  let datasourceFile = datasourceFiles.find(f => {
+    const base = path.basename(f, path.extname(f));
+    return base === datasourceKey || base.includes(datasourceKey);
+  });
+  if (!datasourceFile) {
+    datasourceFile = findDatasourceFileByKey(appPath, schemaBasePath, datasourceFiles, datasourceKey);
+  }
+  if (!datasourceFile) {
+    throw new Error(`Datasource '${datasourceKey}' not found in application config`);
+  }
+  const datasourcePath = path.isAbsolute(schemaBasePath)
+    ? path.join(schemaBasePath, datasourceFile)
+    : path.join(appPath, schemaBasePath, datasourceFile);
+  const datasource = loadConfigFile(datasourcePath);
+  if (datasource.key !== datasourceKey) {
+    throw new Error(`Datasource key mismatch: file has '${datasource.key}', expected '${datasourceKey}'`);
+  }
+  return { datasource, datasourcePath };
+}
+
+module.exports = { loadDatasourceForApp };

package/lib/datasource/unified-validation-run.js

@@ -0,0 +1,92 @@
+const { formatSuccessLine } = require('../utils/cli-test-layout-chalk');
+/**
+ * @fileoverview Run one persisted datasource through the dataplane unified validation flow (POST + optional poll via `lib/api/validation-run.api.js`).
+ *
+ * **CLI:** `aifabrix datasource test`, `test-integration`, and `test-e2e` use this module with different `runType` values.
+ * **User-facing permissions:** see `docs/commands/permissions.md` (Dataplane scopes may differ from the minimum noted on the HTTP helper JSDoc).
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const chalk = require('chalk');
+const { resolveAppKeyForDatasource } = require('./resolve-app');
+const { setupIntegrationTestAuth } = require('../external-system/test-auth');
+const { getConfig } = require('../core/config');
+const { getSystemKeyFromAppKey, findDatasourceFileByKey } = require('./integration-context');
+const { loadDatasourceForApp } = require('./unified-validation-run-resolve');
+const { buildUnifiedValidationBody } = require('./unified-validation-run-body');
+const { postValidationRunAndOptionalPoll } = require('./unified-validation-run-post');
+const { publishDatasourceViaPipeline } = require('../api/pipeline.api');
+const { requireBearerForDataplanePipeline } = require('../utils/token-manager');
+const { formatApiError } = require('../utils/api-error-handler');
+const logger = require('../utils/logger');
+
+/**
+ * Resolve datasource JSON path and loaded object (same rules as test-integration).
+ * Re-export for tests.
+ */
+function loadDatasourceForAppExport(appKey, datasourceKey) {
+  return loadDatasourceForApp(appKey, datasourceKey, findDatasourceFileByKey);
+}
+
+/**
+ * Run unified validation for one persisted datasource.
+ * @async
+ * @param {string} datasourceKey
+ * @param {Object} options
+ * @param {boolean} [options.sync] - Publish local datasource JSON via pipeline before POST validation/run
+ * @returns {Promise<{ envelope: Object|null, apiError: Object|null, pollTimedOut: boolean, incompleteNoAsync: boolean }>}
+ */
+async function runUnifiedDatasourceValidation(datasourceKey, options) {
+  if (!datasourceKey || typeof datasourceKey !== 'string') {
+    throw new Error('Datasource key is required');
+  }
+  const runType = options.runType || 'test';
+  const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
+  const systemKey = await getSystemKeyFromAppKey(appKey);
+  const { datasource } = loadDatasourceForAppExport(appKey, datasourceKey);
+
+  const configObj = await getConfig();
+  const { authConfig, dataplaneUrl } = await setupIntegrationTestAuth(appKey, options, configObj);
+
+  if (options.sync === true) {
+    requireBearerForDataplanePipeline(authConfig);
+    logger.log(chalk.cyan('Syncing local config to dataplane…'));
+    const publishResponse = await publishDatasourceViaPipeline(dataplaneUrl, systemKey, authConfig, datasource);
+    if (!publishResponse || publishResponse.success === false) {
+      const msg =
+        (publishResponse && (publishResponse.formattedError || publishResponse.error)) ||
+        formatApiError(publishResponse, dataplaneUrl) ||
+        'Publish failed';
+      throw new Error(`Sync failed: ${msg}`);
+    }
+    logger.log(formatSuccessLine('Sync complete'));
+  }
+
+  const useAsync = options.noAsync ? false : options.async !== false;
+  const timeoutMs = parseInt(String(options.timeout || '30000'), 10) || 30000;
+
+  const body = await buildUnifiedValidationBody({
+    systemKey,
+    datasourceKey,
+    runType,
+    datasource,
+    payloadPath: options.payload,
+    useAsync,
+    options
+  });
+
+  return postValidationRunAndOptionalPoll({
+    dataplaneUrl,
+    authConfig,
+    body,
+    timeoutMs,
+    useAsync,
+    noAsync: options.noAsync === true || options.async === false
+  });
+}
+
+module.exports = {
+  runUnifiedDatasourceValidation,
+  loadDatasourceForApp: loadDatasourceForAppExport
+};