@aifabrix/builder 2.40.2 → 2.42.0

package/lib/commands/upload.js

@@ -1,25 +1,29 @@
 /**
- * Upload external system to dataplane (upload → validate → publish).
+ * Upload external system to dataplane (single pipeline upload: upload → validate → publish).
  *
  * @fileoverview Upload command handler for aifabrix upload <system-key>
  * @author AI Fabrix Team
  * @version 2.0.0
  */
 
+const path = require('path');
 const chalk = require('chalk');
 const logger = require('../utils/logger');
 const { resolveControllerUrl } = require('../utils/controller-url');
 const { getDeploymentAuth, requireBearerForDataplanePipeline } = require('../utils/token-manager');
 const { resolveDataplaneUrl } = require('../utils/dataplane-resolver');
+const { getIntegrationPath } = require('../utils/paths');
+const { pushCredentialSecrets } = require('../utils/credential-secrets-env');
+const {
+  buildResolvedEnvMapForIntegration,
+  resolveConfigurationValues
+} = require('../utils/configuration-env-resolver');
 const { validateExternalSystemComplete } = require('../validation/validate');
 const { displayValidationResults } = require('../validation/validate-display');
 const { generateControllerManifest } = require('../generator/external-controller-manifest');
-const {
-  uploadApplicationViaPipeline,
-  validateUploadViaPipeline,
-  publishUploadViaPipeline
-} = require('../api/pipeline.api');
+const { uploadApplicationViaPipeline } = require('../api/pipeline.api');
 const { formatApiError } = require('../utils/api-error-handler');
+const { logDataplanePipelineWarning } = require('../utils/dataplane-pipeline-warning');
 
 /**
  * Validates system-key format (same as download).
@@ -37,25 +41,25 @@ function validateSystemKeyFormat(systemKey) {
 
 /**
  * Builds pipeline upload payload from controller manifest.
- * Payload: { version, application, dataSources }; application = system with RBAC.
+ * Payload: { version, application, dataSources, status }; Builder always uses status "draft".
  * @param {Object} manifest - Controller manifest from generateControllerManifest
- * @returns {Object} { version, application, dataSources }
+ * @returns {Object} { version, application, dataSources, status: "draft" }
  */
 function buildUploadPayload(manifest) {
   return {
     version: manifest.version || '1.0.0',
     application: manifest.system,
-    dataSources: manifest.dataSources || []
+    dataSources: manifest.dataSources || [],
+    status: 'draft'
   };
 }
 
 /**
  * Resolves dataplane URL and auth (same pattern as download).
  * @param {string} systemKey - System key
- * @param {Object} options - Options with optional dataplane override
  * @returns {Promise<{ dataplaneUrl: string, authConfig: Object, environment: string }>}
  */
-async function resolveDataplaneAndAuth(systemKey, options) {
+async function resolveDataplaneAndAuth(systemKey) {
   const { resolveEnvironment } = require('../core/config');
   const environment = await resolveEnvironment();
   const controllerUrl = await resolveControllerUrl();
@@ -65,42 +69,42 @@ async function resolveDataplaneAndAuth(systemKey, options) {
     throw new Error('Authentication required. Run "aifabrix login" or "aifabrix app register <system-key>" first.');
   }
 
-  let dataplaneUrl;
-  if (options.dataplane) {
-    dataplaneUrl = options.dataplane.replace(/\/$/, '');
-  } else {
-    logger.log(chalk.blue('Resolving dataplane URL...'));
-    dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig);
-  }
-
+  logger.log(chalk.blue('Resolving dataplane URL...'));
+  const dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig);
   return { dataplaneUrl, authConfig, environment };
 }
 
 /**
- * Runs upload → validate → publish on the dataplane.
+ * Runs single pipeline upload (upload → validate → publish) on the dataplane.
  * @param {string} dataplaneUrl - Dataplane base URL
  * @param {Object} authConfig - Auth config
- * @param {Object} payload - { version, application, dataSources }
- * @returns {Promise<{ uploadId: string }>}
+ * @param {Object} payload - { version, application, dataSources, status: "draft" }
+ * @returns {Promise<Object>} Publication result from Dataplane
  */
 async function runUploadValidatePublish(dataplaneUrl, authConfig, payload) {
-  const uploadRes = await uploadApplicationViaPipeline(dataplaneUrl, authConfig, payload);
-  const uploadId = uploadRes?.data?.uploadId ?? uploadRes?.data?.id ?? uploadRes?.uploadId;
-  if (!uploadId) {
-    const msg = uploadRes?.success === false
-      ? formatApiError(uploadRes, dataplaneUrl)
-      : 'Upload did not return an upload ID';
+  const res = await uploadApplicationViaPipeline(dataplaneUrl, authConfig, payload);
+  if (res?.success === false) {
+    const msg = formatApiError(res, dataplaneUrl);
     throw new Error(msg);
   }
+  return res;
+}
 
-  const validateRes = await validateUploadViaPipeline(dataplaneUrl, uploadId, authConfig);
-  if (validateRes?.success === false) {
-    const msg = formatApiError(validateRes, dataplaneUrl);
-    throw new Error(`Upload validation failed: ${msg}`);
+/**
+ * Builds a short summary of validation errors for the thrown message.
+ * @param {Object} validationResult - Result from validateExternalSystemComplete
+ * @returns {string} First few errors joined for the error message
+ */
+function formatValidationErrorSummary(validationResult) {
+  const errors = validationResult.errors || [];
+  if (errors.length === 0) {
+    return 'Validation failed. Fix errors before uploading.';
   }
-
-  await publishUploadViaPipeline(dataplaneUrl, uploadId, authConfig);
-  return { uploadId };
+  const maxShow = 3;
+  const shown = errors.slice(0, maxShow).map(e => (typeof e === 'string' ? e : String(e)));
+  const summary = shown.join('; ');
+  const more = errors.length > maxShow ? ` (and ${errors.length - maxShow} more)` : '';
+  return `Validation failed: ${summary}${more}. Fix errors above and run the command again.`;
 }
 
 /**
@@ -111,22 +115,62 @@ async function runUploadValidatePublish(dataplaneUrl, authConfig, payload) {
 function throwIfValidationFailed(validationResult) {
   if (!validationResult.valid) {
     displayValidationResults(validationResult);
-    throw new Error('Validation failed. Fix errors before uploading.');
+    throw new Error(formatValidationErrorSummary(validationResult));
   }
 }
 
 /**
- * Uploads external system to dataplane (upload → validate → publish). No controller deploy.
+ * Resolves configuration arrays in the upload payload (application + dataSources) from env and secrets.
+ * @param {string} systemKey - System key
+ * @param {Object} payload - Upload payload (mutated)
+ */
+async function resolvePayloadConfiguration(systemKey, payload) {
+  const { envMap, secrets } = await buildResolvedEnvMapForIntegration(systemKey);
+  if (Array.isArray(payload.application?.configuration) && payload.application.configuration.length > 0) {
+    resolveConfigurationValues(payload.application.configuration, envMap, secrets, systemKey);
+  }
+  for (const ds of payload.dataSources || []) {
+    if (Array.isArray(ds?.configuration) && ds.configuration.length > 0) {
+      resolveConfigurationValues(ds.configuration, envMap, secrets, systemKey);
+    }
+  }
+}
+
+/**
+ * Pushes credential secrets from .env and payload to dataplane; logs result or warning.
+ * @param {string} dataplaneUrl - Dataplane URL
+ * @param {Object} authConfig - Auth config
+ * @param {string} systemKey - System key
+ * @param {Object} payload - Upload payload
+ */
+async function pushAndLogCredentialSecrets(dataplaneUrl, authConfig, systemKey, payload) {
+  const envFilePath = path.join(getIntegrationPath(systemKey), '.env');
+  const pushResult = await pushCredentialSecrets(dataplaneUrl, authConfig, {
+    envFilePath,
+    appName: systemKey,
+    payload
+  });
+  if (pushResult.pushed > 0) {
+    const keyList = pushResult.keys?.length ? ` (${pushResult.keys.join(', ')})` : '';
+    logger.log(chalk.green(`Pushed ${pushResult.pushed} credential secret(s) to dataplane${keyList}.`));
+  } else {
+    logger.log(chalk.yellow('Secret push skipped'));
+  }
+  if (pushResult.warning) {
+    logger.log(chalk.yellow(`Warning: ${pushResult.warning}`));
+  }
+}
+
+/**
+ * Uploads external system to dataplane (single pipeline upload). No controller deploy.
  * @param {string} systemKey - External system key (integration/<system-key>/)
  * @param {Object} [options] - Options
  * @param {boolean} [options.dryRun] - Validate and build payload only; no API calls
- * @param {string} [options.dataplane] - Override dataplane URL
  * @returns {Promise<void>}
  * @throws {Error} If validation or API calls fail
  */
 async function uploadExternalSystem(systemKey, options = {}) {
   validateSystemKeyFormat(systemKey);
-
   logger.log(chalk.blue(`\nUploading external system to dataplane: ${systemKey}`));
 
   const validationResult = await validateExternalSystemComplete(systemKey, { type: 'external' });
@@ -135,6 +179,7 @@ async function uploadExternalSystem(systemKey, options = {}) {
 
   const manifest = await generateControllerManifest(systemKey, { type: 'external' });
   const payload = buildUploadPayload(manifest);
+  await resolvePayloadConfiguration(systemKey, payload);
 
   if (options.dryRun) {
     logger.log(chalk.yellow('Dry run: would upload payload (no API calls).'));
@@ -142,12 +187,23 @@ async function uploadExternalSystem(systemKey, options = {}) {
     return;
   }
 
-  const { dataplaneUrl, authConfig, environment } = await resolveDataplaneAndAuth(systemKey, options);
+  const { dataplaneUrl, authConfig, environment } = await resolveDataplaneAndAuth(systemKey);
   requireBearerForDataplanePipeline(authConfig);
   logger.log(chalk.blue(`Dataplane: ${dataplaneUrl}`));
+  logDataplanePipelineWarning();
 
+  await pushAndLogCredentialSecrets(dataplaneUrl, authConfig, systemKey, payload);
   await runUploadValidatePublish(dataplaneUrl, authConfig, payload);
+  logUploadSuccess(environment, systemKey, dataplaneUrl);
+}
 
+/**
+ * Logs upload success summary.
+ * @param {string} environment - Environment key
+ * @param {string} systemKey - System key
+ * @param {string} dataplaneUrl - Dataplane URL
+ */
+function logUploadSuccess(environment, systemKey, dataplaneUrl) {
   logger.log(chalk.green('\nUpload validated and published to dataplane.'));
   logger.log(chalk.blue(`Environment: ${environment}`));
   logger.log(chalk.blue(`System: ${systemKey}`));

package/lib/commands/wizard-core-helpers.js

@@ -6,8 +6,14 @@
 
 const chalk = require('chalk');
 const ora = require('ora');
+const path = require('path');
+const fs = require('fs').promises;
+const yaml = require('js-yaml');
 const logger = require('../utils/logger');
+const { getIntegrationPath } = require('../utils/paths');
 const { parseOpenApi, testMcpConnection, credentialSelection } = require('../api/wizard.api');
+const { listCredentials } = require('../api/credentials.api');
+const { listExternalSystems, getExternalSystem } = require('../api/external-systems.api');
 
 /**
  * Parse OpenAPI file or URL
@@ -158,6 +164,7 @@ function buildConfigPreferences(configPrefs) {
     intent: configPrefs?.intent || 'general integration',
     fieldOnboardingLevel: configPrefs?.fieldOnboardingLevel || 'full',
     enableOpenAPIGeneration: configPrefs?.enableOpenAPIGeneration !== false,
+    debug: configPrefs?.debug === true,
     userPreferences: {
       enableMCP: configPrefs?.enableMCP || false,
       enableABAC: configPrefs?.enableABAC || false,
@@ -175,9 +182,10 @@ function buildConfigPreferences(configPrefs) {
  * @param {Object} params.prefs - Configuration preferences
  * @param {string} [params.credentialIdOrKey] - Credential ID or key
  * @param {string} [params.systemIdOrKey] - System ID or key
+ * @param {string} [params.entityName] - Entity name for multi-entity OpenAPI (from discover-entities)
  * @returns {Object} Configuration payload
  */
-function buildConfigPayload({ openapiSpec, detectedType, mode, prefs, credentialIdOrKey, systemIdOrKey }) {
+function buildConfigPayload({ openapiSpec, detectedType, mode, prefs, credentialIdOrKey, systemIdOrKey, entityName }) {
   const detectedTypeValue = detectedType?.recommendedType || detectedType?.apiType || detectedType?.selectedType || 'record-based';
   const payload = {
     openapiSpec,
@@ -190,6 +198,32 @@ function buildConfigPayload({ openapiSpec, detectedType, mode, prefs, credential
   };
   if (credentialIdOrKey) payload.credentialIdOrKey = credentialIdOrKey;
   if (systemIdOrKey) payload.systemIdOrKey = systemIdOrKey;
+  if (entityName) payload.entityName = entityName;
+  if (prefs.debug) payload.debug = true;
+  return payload;
+}
+
+/**
+ * Build payload for platform config endpoint.
+ * Schema allows only: datasourceKeys?, credentialIdOrKey?, configurationValues?, debug?
+ * (additionalProperties: false - do NOT send intent, mode, fieldOnboardingLevel, etc.)
+ * @param {Object} params - Parameters object
+ * @param {string} [params.credentialIdOrKey] - Credential ID or key
+ * @param {string[]} [params.datasourceKeys] - Datasource keys to include (defaults to all)
+ * @param {Object} [params.configurationValues] - Configuration value overrides
+ * @param {boolean} [params.debug] - When true, capture debug log
+ * @returns {Object} Platform config payload
+ */
+function buildPlatformConfigPayload({ credentialIdOrKey, datasourceKeys, configurationValues, debug }) {
+  const payload = {};
+  if (credentialIdOrKey) payload.credentialIdOrKey = credentialIdOrKey;
+  if (datasourceKeys && Array.isArray(datasourceKeys) && datasourceKeys.length > 0) {
+    payload.datasourceKeys = datasourceKeys;
+  }
+  if (configurationValues && typeof configurationValues === 'object' && Object.keys(configurationValues).length > 0) {
+    payload.configurationValues = configurationValues;
+  }
+  if (debug) payload.debug = true;
   return payload;
 }
 
@@ -250,14 +284,19 @@ function formatValidationDetailsPlain(errorData) {
 /**
  * Create and throw config generation error with optional formatted message
  * @param {Object} generateResponse - API response (error)
+ * @param {Object} [options] - Optional options
+ * @param {string} [options.debugManifestHint] - Hint to append when debug manifest was saved (e.g. review debug.log and fix manually)
  * @throws {Error}
  */
-function throwConfigGenerationError(generateResponse) {
+function throwConfigGenerationError(generateResponse, options = {}) {
   const summary = generateResponse.error || generateResponse.formattedError || 'Unknown error';
   const detailsPlain = formatValidationDetailsPlain(generateResponse.errorData);
-  const message = detailsPlain
+  let message = detailsPlain
     ? `Configuration generation failed: ${summary}\n${detailsPlain}`
     : `Configuration generation failed: ${summary}`;
+  if (options.debugManifestHint) {
+    message += `\n\n${options.debugManifestHint}`;
+  }
   const err = new Error(message);
   if (generateResponse.formattedError) {
     err.formatted = generateResponse.formattedError;
@@ -265,6 +304,181 @@ function throwConfigGenerationError(generateResponse) {
   throw err;
 }
 
+/**
+ * Write debug log to integration/<appName>/debug.log
+ * @async
+ * @param {string} appName - Application name
+ * @param {string} content - Debug log content
+ */
+async function writeDebugLog(appName, content) {
+  try {
+    const dir = getIntegrationPath(appName);
+    await fs.mkdir(dir, { recursive: true });
+    const debugPath = path.join(dir, 'debug.log');
+    await fs.writeFile(debugPath, content, 'utf8');
+    logger.log(chalk.gray(`  Debug log saved to integration/${appName}/debug.log`));
+  } catch (e) {
+    logger.warn(`Could not save debug.log: ${e.message}`);
+  }
+}
+
+/**
+ * Write systemConfig and datasourceConfig from error response for manual fix
+ * @async
+ * @param {string} appName - Application name
+ * @param {Object} [systemConfig] - System config from errorData
+ * @param {Object|Object[]} [datasourceConfig] - Datasource config(s) from errorData
+ * @returns {Promise<string[]>} Names of saved files
+ */
+async function writeDebugManifest(appName, systemConfig, datasourceConfig) {
+  const saved = [];
+  try {
+    const dir = getIntegrationPath(appName);
+    await fs.mkdir(dir, { recursive: true });
+    if (systemConfig && typeof systemConfig === 'object') {
+      const systemPath = path.join(dir, 'debug-system.yaml');
+      await fs.writeFile(systemPath, yaml.dump(systemConfig, { lineWidth: -1 }), 'utf8');
+      saved.push('debug-system.yaml');
+      logger.log(chalk.gray(`  Debug manifest saved to integration/${appName}/debug-system.yaml`));
+    }
+    if (datasourceConfig !== undefined && datasourceConfig !== null) {
+      const configs = Array.isArray(datasourceConfig) ? datasourceConfig : [datasourceConfig];
+      if (configs.length > 0 && configs.every(c => c && typeof c === 'object')) {
+        const datasourcePath = path.join(dir, 'debug-datasource.yaml');
+        const toWrite = configs.length === 1 ? configs[0] : configs;
+        await fs.writeFile(datasourcePath, yaml.dump(toWrite, { lineWidth: -1 }), 'utf8');
+        saved.push('debug-datasource.yaml');
+        logger.log(chalk.gray(`  Debug manifest saved to integration/${appName}/debug-datasource.yaml`));
+      }
+    }
+  } catch (e) {
+    logger.warn(`Could not save debug manifest: ${e.message}`);
+  }
+  return saved;
+}
+
+/**
+ * Save debug manifest on error and throw
+ * @param {Object} generateResponse - API error response
+ * @param {Object} opts - Options
+ * @param {boolean} opts.enableDebug - Whether debug was enabled
+ * @param {string} [opts.appName] - App name for writing files
+ */
+async function saveDebugManifestOnErrorAndThrow(generateResponse, opts) {
+  const { enableDebug, appName } = opts;
+  let debugManifestHint = null;
+  if (enableDebug && appName) {
+    const errorData = generateResponse.errorData || {};
+    const debugLog = errorData.debugLog;
+    if (debugLog && typeof debugLog === 'string') {
+      await writeDebugLog(appName, debugLog);
+    }
+    const systemConfig = errorData.systemConfig;
+    const datasourceConfig = errorData.datasourceConfig || errorData.datasourceConfigs;
+    const savedManifest = await writeDebugManifest(appName, systemConfig, datasourceConfig);
+    if (debugLog || savedManifest.length > 0) {
+      const files = [debugLog && 'debug.log', ...savedManifest].filter(Boolean).join(', ');
+      debugManifestHint = `Debug manifest saved to integration/${appName}/ (${files}). Review the log and fix the manifest manually, then run: aifabrix wizard ${appName}`;
+    }
+  }
+  throwConfigGenerationError(generateResponse, { debugManifestHint });
+}
+
+/** Throws with validation error; saves debug manifest when options.debug and options.appName are set. */
+async function throwValidationFailureWithDebug(validateResponse, systemConfig, configs, errorMsg, options) {
+  if (!options.debug || !options.appName) throw new Error(`Configuration validation failed: ${errorMsg}`);
+  const errorData = validateResponse.errorData || validateResponse.data || {};
+  const debugLog = errorData.debugLog;
+  if (debugLog && typeof debugLog === 'string') await writeDebugLog(options.appName, debugLog);
+  const savedManifest = await writeDebugManifest(
+    options.appName, errorData.systemConfig || systemConfig, errorData.datasourceConfig || errorData.datasourceConfigs || configs
+  );
+  if (!debugLog && savedManifest.length === 0) throw new Error(`Configuration validation failed: ${errorMsg}`);
+  const files = [debugLog && 'debug.log', ...savedManifest].filter(Boolean).join(', ');
+  throw new Error(`Configuration validation failed: ${errorMsg}\n\nDebug manifest saved to integration/${options.appName}/ (${files}). Review the log and fix the manifest manually, then run: aifabrix wizard ${options.appName}`);
+}
+
+/**
+ * Resolve credential config from user action (select/skip/create)
+ * @async
+ * @param {string} dataplaneUrl - Dataplane URL
+ * @param {Object} authConfig - Auth config
+ * @param {Object} credentialAction - Result from promptForCredentialAction
+ * @returns {Promise<Object>} configCredential for handleCredentialSelection
+ */
+async function resolveCredentialConfig(dataplaneUrl, authConfig, credentialAction) {
+  const { promptForExistingCredential } = require('../generator/wizard-prompts');
+  if (credentialAction.action !== 'select') {
+    return credentialAction.action === 'skip' ? { action: 'skip' } : { action: credentialAction.action };
+  }
+  let credentialsList = [];
+  try {
+    const listResponse = await listCredentials(dataplaneUrl, authConfig, {
+      activeOnly: true,
+      pageSize: 100
+    });
+    const body = listResponse?.data ?? listResponse;
+    const inner = Array.isArray(body) ? body : (body?.data ?? body);
+    credentialsList = Array.isArray(inner) ? inner : (inner?.credentials ?? inner?.items ?? []) || [];
+  } catch (_) {
+    credentialsList = [];
+  }
+  const selected = await promptForExistingCredential(Array.isArray(credentialsList) ? credentialsList : []);
+  return { action: 'select', credentialIdOrKey: selected.credentialIdOrKey };
+}
+
+/**
+ * Fetch external systems list from dataplane
+ * @async
+ * @param {string} dataplaneUrl - Dataplane URL
+ * @param {Object} authConfig - Auth config
+ * @returns {Promise<Object[]>} Systems list
+ */
+async function fetchSystemsListForAddDatasource(dataplaneUrl, authConfig) {
+  try {
+    const listResponse = await listExternalSystems(dataplaneUrl, authConfig, { pageSize: 100 });
+    const body = listResponse?.data ?? listResponse;
+    const inner = Array.isArray(body) ? body : (body?.data ?? body);
+    const list = Array.isArray(inner) ? inner : (inner?.items ?? inner?.systems ?? []) || [];
+    return Array.isArray(list) ? list : [];
+  } catch (_) {
+    return [];
+  }
+}
+
+/**
+ * Resolve and validate external system for add-datasource (prompt until valid)
+ * @async
+ * @param {string} dataplaneUrl - Dataplane URL
+ * @param {Object} authConfig - Auth config
+ * @param {Object[]} systemsList - Systems list
+ * @param {string} initialSystemIdOrKey - Initial system ID or key
+ * @returns {Promise<{systemResponse: Object, systemIdOrKey: string}>}
+ */
+async function resolveExternalSystemForAddDatasource(dataplaneUrl, authConfig, systemsList, initialSystemIdOrKey) {
+  const { promptForExistingSystem } = require('../generator/wizard-prompts');
+  const { isExternalSystemForAddDatasource } = require('./wizard-helpers');
+  let systemIdOrKey = initialSystemIdOrKey;
+  let systemResponse;
+  for (;;) {
+    try {
+      systemResponse = await getExternalSystem(dataplaneUrl, systemIdOrKey, authConfig);
+      const sys = systemResponse?.data || systemResponse;
+      if (sys && (systemResponse?.data || systemResponse?.success)) {
+        if (!isExternalSystemForAddDatasource(sys)) {
+          logger.log(chalk.red('Cannot add datasource to a webapp. Please select an external system.'));
+        } else {
+          break;
+        }
+      }
+    } catch (err) {
+      logger.log(chalk.red(`System not found or error: ${err.message}`));
+    }
+    systemIdOrKey = await promptForExistingSystem(systemsList, systemIdOrKey);
+  }
+  return { systemResponse, systemIdOrKey };
+}
+
 module.exports = {
   parseOpenApiSource,
   testMcpServerConnection,
@@ -272,7 +486,15 @@ module.exports = {
   runCredentialSelectionLoop,
   buildConfigPreferences,
   buildConfigPayload,
+  buildPlatformConfigPayload,
   extractConfigurationFromResponse,
   formatValidationDetailsPlain,
-  throwConfigGenerationError
+  throwConfigGenerationError,
+  writeDebugLog,
+  writeDebugManifest,
+  saveDebugManifestOnErrorAndThrow,
+  throwValidationFailureWithDebug,
+  resolveCredentialConfig,
+  fetchSystemsListForAddDatasource,
+  resolveExternalSystemForAddDatasource
 };