@aifabrix/builder 2.41.0 → 2.42.1

package/lib/external-system/download.js

@@ -1,28 +1,37 @@
 /**
  * External System Download Module
  *
- * Downloads external systems from dataplane to local development structure.
- * Supports downloading system configuration and datasources for local development.
+ * Downloads full running manifest from dataplane and splits it into component files
+ * using split-json. The dataplane GET /api/v1/external/systems/{systemKey}/config
+ * returns { application, dataSources, version } in pipeline format. Supports legacy
+ * responses where datasources are inline in application.configuration.dataSources.
+ *
+ * Process: validate key → auth + dataplane (Bearer required) → GET full manifest →
+ * validate → build deploy JSON (augments configuration with auth KV_* for env.template) →
+ * write deploy JSON → splitDeployJson (merge env.template if exists; README overwrite per
+ * prompt or --force) → ensure placeholder secrets from env.template → convert to JSON if
+ * --format json.
  *
  * @fileoverview External system download functionality for AI Fabrix Builder
  * @author AI Fabrix Team
  * @version 2.0.0
+ * @see docs/commands/external-integration.md#aifabrix-download-system-key
  */
 
 const fs = require('fs').promises;
+const fsSync = require('fs');
 const path = require('path');
-const os = require('os');
+const readline = require('readline');
 const yaml = require('js-yaml');
 const chalk = require('chalk');
 const { getExternalSystemConfig } = require('../api/external-systems.api');
-const { getDeploymentAuth } = require('../utils/token-manager');
+const { getDeploymentAuth, requireBearerForDataplanePipeline } = require('../utils/token-manager');
 const { getConfig } = require('../core/config');
-const { detectAppType } = require('../utils/paths');
+const { getIntegrationPath } = require('../utils/paths');
+const { retemplateConfigurationForDownload } = require('../utils/configuration-env-resolver');
 const logger = require('../utils/logger');
-const { writeConfigFile } = require('../utils/config-format');
-const { generateEnvTemplate } = require('../utils/external-system-env-helpers');
-const { generateVariablesYaml, generateReadme } = require('./download-helpers');
 const { resolveControllerUrl } = require('../utils/controller-url');
+const generator = require('../generator');
 
 /**
  * Validates system type from downloaded application
@@ -97,25 +106,6 @@ function validateDownloadedData(application, dataSources) {
   }
 }
 
-/**
- * Handles partial download errors gracefully
- * @param {string} systemKey - System key
- * @param {Object} systemData - System data that was successfully downloaded
- * @param {Array<Error>} datasourceErrors - Array of errors from datasource downloads
- * @throws {Error} Aggregated error message
- */
-function handlePartialDownload(systemKey, systemData, datasourceErrors) {
-  if (datasourceErrors.length === 0) {
-    return;
-  }
-
-  const errorMessages = datasourceErrors.map(err => err.message).join('\n  - ');
-  throw new Error(
-    `Partial download completed for system '${systemKey}', but some datasources failed:\n  - ${errorMessages}\n\n` +
-    'System configuration was downloaded successfully. You may need to download datasources separately.'
-  );
-}
-
 /**
  * Setup authentication and get dataplane URL
  * @async
@@ -131,7 +121,8 @@ async function setupAuthenticationAndDataplane(systemKey, _options, _config) {
   const controllerUrl = await resolveControllerUrl();
   const authConfig = await getDeploymentAuth(controllerUrl, environment, systemKey);
 
-  if (!authConfig.token && !authConfig.clientId) {
+  requireBearerForDataplanePipeline(authConfig);
+  if (!authConfig.token) {
     throw new Error('Authentication required. Run "aifabrix login" or "aifabrix app register" first.');
   }
 
@@ -144,16 +135,19 @@ async function setupAuthenticationAndDataplane(systemKey, _options, _config) {
 }
 
 /**
- * Download system configuration from dataplane
+ * Download full running manifest from dataplane
+ * GET /api/v1/external/systems/{systemKey}/config returns
+ * { application, dataSources, version } in pipeline format.
+ *
  * @async
  * @param {string} dataplaneUrl - Dataplane URL
  * @param {string} systemKey - System key
  * @param {Object} authConfig - Authentication configuration
- * @returns {Promise<Object>} Object with application and dataSources
+ * @returns {Promise<{application: Object, dataSources: Array, version?: string}>} Full manifest
  * @throws {Error} If download fails
  */
-async function downloadSystemConfiguration(dataplaneUrl, systemKey, authConfig) {
-  logger.log(chalk.blue(`📡 Downloading system configuration: ${systemKey}`));
+async function downloadFullManifest(dataplaneUrl, systemKey, authConfig) {
+  logger.log(chalk.blue(`📡 Downloading full manifest: ${systemKey}`));
   const response = await getExternalSystemConfig(dataplaneUrl, systemKey, authConfig);
 
   if (!response.success || !response.data) {
@@ -163,14 +157,14 @@ async function downloadSystemConfiguration(dataplaneUrl, systemKey, authConfig)
   const downloadData = response.data.data || response.data;
   let application = downloadData.application;
   let dataSources = downloadData.dataSources || [];
+  const version = downloadData.version;
 
-  // Handle case where datasources are inline in application.configuration.dataSources
+  // Legacy: datasources inline in application.configuration.dataSources
   if (!application && downloadData.configuration) {
     application = downloadData;
   }
   if (application && application.configuration && Array.isArray(application.configuration.dataSources)) {
     dataSources = [...dataSources, ...application.configuration.dataSources];
-    // Remove inline datasources from application to avoid duplication
     const { dataSources: _inlineDataSources, ...configWithoutDataSources } = application.configuration;
     application = { ...application, configuration: configWithoutDataSources };
   }
@@ -179,149 +173,78 @@ async function downloadSystemConfiguration(dataplaneUrl, systemKey, authConfig)
     throw new Error('Application configuration not found in download response');
   }
 
-  return { application, dataSources };
+  return { application, dataSources, version };
 }
 
 /**
- * Generate files in temporary directory
- * @async
- * @param {string} tempDir - Temporary directory path
- * @param {string} systemKey - System key
- * @param {Object} application - Application configuration
- * @param {Array} dataSources - Array of datasource configurations
- * @returns {Promise<Object>} Object with file paths
- * @throws {Error} If file generation fails
- */
-/**
- * Generates system file
- * @async
- * @function generateSystemFile
- * @param {string} tempDir - Temporary directory
- * @param {string} systemKey - System key
- * @param {Object} application - Application object
- * @returns {Promise<string>} System file path
+ * Derives env variable name from system key and security key (e.g. KV_HUBSPOT_CLIENTID).
+ * @param {string} systemKey - System key (e.g. 'hubspot')
+ * @param {string} securityKey - Security key (e.g. 'clientId')
+ * @returns {string} Variable name
  */
-async function generateSystemFile(tempDir, systemKey, application) {
-  const systemFileName = `${systemKey}-system.yaml`;
-  const systemFilePath = path.join(tempDir, systemFileName);
-  writeConfigFile(systemFilePath, application);
-  return systemFilePath;
+function deriveAuthVarName(systemKey, securityKey) {
+  const safeKey = (systemKey || '').toUpperCase().replace(/-/g, '_');
+  const safeSec = (securityKey || '').replace(/([A-Z])/g, '_$1').toUpperCase().replace(/^_/, '');
+  return `KV_${safeKey}_${safeSec}`;
 }
 
 /**
- * Generates datasource files
- * @async
- * @function generateDatasourceFiles
- * @param {string} tempDir - Temporary directory
- * @param {string} systemKey - System key
- * @param {Array} dataSources - Array of datasource objects
- * @returns {Promise<Object>} Object with datasourceFiles array and datasourceErrors array
+ * Collects kv:// paths already present in configuration array.
+ * @param {Array} config - Configuration array
+ * @returns {Set<string>} Set of kv paths
  */
-async function generateDatasourceFiles(tempDir, systemKey, dataSources) {
-  const datasourceErrors = [];
-  const datasourceFiles = [];
-  for (const datasource of dataSources) {
-    try {
-      const datasourceKey = datasource.key || '';
-      // Extract datasource key (remove system key prefix if present)
-      let datasourceKeyOnly;
-      if (datasourceKey.startsWith(`${systemKey}-`)) {
-        datasourceKeyOnly = datasourceKey.substring(systemKey.length + 1);
-      } else {
-        const entityType = datasource.entityType || datasource.entityKey || datasourceKey.split('-').pop();
-        datasourceKeyOnly = entityType;
-      }
-      const datasourceFileName = `${systemKey}-datasource-${datasourceKeyOnly}.yaml`;
-      const datasourceFilePath = path.join(tempDir, datasourceFileName);
-      writeConfigFile(datasourceFilePath, datasource);
-      datasourceFiles.push(datasourceFilePath);
-    } catch (error) {
-      datasourceErrors.push(new Error(`Failed to write datasource ${datasource.key}: ${error.message}`));
-    }
+function collectExistingKvPaths(config) {
+  const paths = new Set();
+  for (const item of config) {
+    if (!item || !item.value) continue;
+    const val = String(item.value).trim();
+    if (val.startsWith('kv://')) paths.add(val);
+    else if (item.location === 'keyvault') paths.add(`kv://${val}`);
   }
-  return { datasourceFiles, datasourceErrors };
+  return paths;
 }
 
 /**
- * Generates configuration files (application.yaml, env.template, README.md)
- * @async
- * @function generateConfigFiles
- * @param {string} tempDir - Temporary directory
- * @param {string} systemKey - System key
- * @param {Object} application - Application object
- * @param {Array} dataSources - Array of datasource objects
- * @returns {Promise<Object>} Object with file paths
+ * Augments system.configuration with entries for authentication.security kv paths
+ * so that extractEnvTemplate produces env.template that passes validateAuthKvCoverage.
+ * @param {Object} system - System config (mutated)
  */
-async function generateConfigFiles(tempDir, systemKey, application, dataSources) {
-  // Generate application.yaml
-  const variables = generateVariablesYaml(systemKey, application, dataSources);
-  const variablesPath = path.join(tempDir, 'application.yaml');
-  await fs.writeFile(variablesPath, yaml.dump(variables, { indent: 2, lineWidth: 120, noRefs: true }), 'utf8');
-
-  // Generate env.template
-  const envTemplate = generateEnvTemplate(application);
-  const envTemplatePath = path.join(tempDir, 'env.template');
-  await fs.writeFile(envTemplatePath, envTemplate, 'utf8');
-
-  // Generate README.md
-  const readme = generateReadme(systemKey, application, dataSources);
-  const readmePath = path.join(tempDir, 'README.md');
-  await fs.writeFile(readmePath, readme, 'utf8');
-
-  return { variablesPath, envTemplatePath, readmePath };
-}
-
-async function generateFilesInTempDir(tempDir, systemKey, application, dataSources) {
-  const systemFilePath = await generateSystemFile(tempDir, systemKey, application);
-
-  const { datasourceFiles, datasourceErrors } = await generateDatasourceFiles(tempDir, systemKey, dataSources);
-
-  // Handle partial downloads
-  if (datasourceErrors.length > 0) {
-    handlePartialDownload(systemKey, application, datasourceErrors);
+function augmentConfigurationWithAuthSecrets(system) {
+  if (!system || typeof system !== 'object') return;
+  const security = system.authentication?.security;
+  if (!security || typeof security !== 'object') return;
+  const config = Array.isArray(system.configuration) ? [...system.configuration] : [];
+  const existingPaths = collectExistingKvPaths(config);
+  const systemKey = system.key || '';
+  for (const [key, val] of Object.entries(security)) {
+    if (typeof val !== 'string' || !/^kv:\/\/.+/.test(val)) continue;
+    if (existingPaths.has(val)) continue;
+    config.push({
+      name: deriveAuthVarName(systemKey, key),
+      value: val.replace(/^kv:\/\//, ''),
+      location: 'keyvault',
+      required: true
+    });
   }
-
-  const { variablesPath, envTemplatePath, readmePath } = await generateConfigFiles(tempDir, systemKey, application, dataSources);
-
-  return {
-    systemFilePath,
-    variablesPath,
-    envTemplatePath,
-    readmePath,
-    datasourceFiles
-  };
+  system.configuration = config;
 }
 
 /**
- * Move files from temporary directory to final location
- * @async
- * @param {string} tempDir - Temporary directory path
- * @param {string} finalPath - Final destination path
- * @param {string} systemKey - System key
- * @param {Object} filePaths - Object with file paths
- * @throws {Error} If file move fails
+ * Build deploy JSON from full running manifest (split-json compatible).
+ * @param {Object} application - System config from dataplane
+ * @param {Array} dataSources - Inline datasource configs
+ * @param {string} [version] - Optional version
+ * @returns {Object} Deploy JSON object for splitDeployJson
  */
-async function moveFilesToFinalLocation(tempDir, finalPath, systemKey, filePaths) {
-  logger.log(chalk.blue(`📁 Creating directory: ${finalPath}`));
-  await fs.mkdir(finalPath, { recursive: true });
-
-  const systemFileName = `${systemKey}-system.json`;
-  const filesToMove = [
-    { from: filePaths.systemFilePath, to: path.join(finalPath, systemFileName) },
-    { from: filePaths.variablesPath, to: path.join(finalPath, 'application.yaml') },
-    { from: filePaths.envTemplatePath, to: path.join(finalPath, 'env.template') },
-    { from: filePaths.readmePath, to: path.join(finalPath, 'README.md') }
-  ];
-
-  for (const dsFile of filePaths.datasourceFiles) {
-    const fileName = path.basename(dsFile);
-    filesToMove.push({ from: dsFile, to: path.join(finalPath, fileName) });
-  }
-
-  for (const file of filesToMove) {
-    await fs.copyFile(file.from, file.to);
-    logger.log(chalk.green(`✓ Created: ${path.relative(process.cwd(), file.to)}`));
-  }
+function buildDeployJsonFromManifest(application, dataSources, version) {
+  const dataSourcesKeys = Array.isArray(dataSources)
+    ? dataSources.map(ds => (ds && ds.key) ? ds.key : ds)
+    : [];
+  const system = { ...application, dataSources: application.dataSources || dataSourcesKeys };
+  augmentConfigurationWithAuthSecrets(system);
+  const deploy = { system, dataSources };
+  if (version) deploy.version = version;
+  return deploy;
 }
 
 /**
@@ -346,8 +269,8 @@ function validateSystemKeyFormat(systemKey) {
 function handleDryRun(systemKey, dataplaneUrl) {
   logger.log(chalk.yellow('🔍 Dry run mode - would download from:'));
   logger.log(chalk.gray(`  ${dataplaneUrl}/api/v1/external/systems/${systemKey}/config`));
-  logger.log(chalk.yellow('\nWould create:'));
-  logger.log(chalk.gray(`  integration/${systemKey}/`));
+  logger.log(chalk.yellow('\nWould create (via split-json):'));
+  logger.log(chalk.gray(`  integration/${systemKey}/${systemKey}-deploy.json`));
   logger.log(chalk.gray(`  integration/${systemKey}/application.yaml`));
   logger.log(chalk.gray(`  integration/${systemKey}/${systemKey}-system.yaml`));
   logger.log(chalk.gray(`  integration/${systemKey}/env.template`));
@@ -370,25 +293,85 @@ function validateAndLogDownloadedData(application, dataSources) {
 }
 
 /**
- * Process downloaded system (generate files, move, cleanup)
+ * Prompts user: "Do you want to replace README.md? (yes/no)"
+ * @returns {Promise<boolean>} True if user answers yes
+ */
+function promptReplaceReadme() {
+  const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+  return new Promise(resolve => {
+    rl.question(chalk.yellow('README.md already exists. Do you want to replace it? (yes/no) '), answer => {
+      rl.close();
+      const normalized = (answer || '').trim().toLowerCase();
+      resolve(normalized === 'yes' || normalized === 'y');
+    });
+  });
+}
+
+/**
+ * Resolves split options for download: merge env.template if it exists, prompt for README replace if it exists (unless force).
+ * @param {string} finalPath - Integration directory path
+ * @param {Object} [options] - Download options
+ * @param {boolean} [options.force] - If true, overwrite README.md without prompting
+ * @returns {Promise<{ mergeEnvTemplate: boolean, overwriteReadme: boolean }>}
+ */
+async function resolveDownloadSplitOptions(finalPath, options = {}) {
+  const opts = { mergeEnvTemplate: false, overwriteReadme: true };
+  if (!fsSync.existsSync(finalPath)) return opts;
+  const envPath = path.join(finalPath, 'env.template');
+  const readmePath = path.join(finalPath, 'README.md');
+  if (fsSync.existsSync(envPath)) opts.mergeEnvTemplate = true;
+  if (fsSync.existsSync(readmePath)) {
+    if (options.force) {
+      opts.overwriteReadme = true;
+    } else {
+      opts.overwriteReadme = await promptReplaceReadme();
+      if (!opts.overwriteReadme) logger.log(chalk.gray('  Keeping existing README.md'));
+    }
+  }
+  return opts;
+}
+
+/**
+ * Re-templates system file configuration from env.template when present (mutates file on disk).
+ * @param {string} systemKey - System key
+ * @param {string} systemFilePath - Path to *-system.yaml
+ * @returns {Promise<void>}
+ */
+async function applyRetemplateToSystemFile(systemKey, systemFilePath) {
+  if (!systemFilePath || !fsSync.existsSync(systemFilePath)) return;
+  const systemContent = await fs.readFile(systemFilePath, 'utf8');
+  const systemObj = yaml.load(systemContent);
+  if (!Array.isArray(systemObj?.configuration)) return;
+  const applied = await retemplateConfigurationForDownload(systemKey, systemObj.configuration);
+  if (applied) {
+    await fs.writeFile(systemFilePath, yaml.dump(systemObj, { indent: 2, lineWidth: -1 }), 'utf8');
+  }
+}
+
+/**
+ * Write deploy JSON and split into component files
  * @async
  * @param {string} systemKey - System key
- * @param {Object} application - Application configuration
- * @param {Array} dataSources - Array of datasource configurations
- * @param {string} tempDir - Temporary directory path
+ * @param {Object} manifest - Full manifest { application, dataSources, version }
+ * @param {Object} [splitOptions] - Options for split (mergeEnvTemplate, overwriteReadme)
  * @returns {Promise<string>} Final destination path
  * @throws {Error} If processing fails
  */
-async function processDownloadedSystem(systemKey, application, dataSources, tempDir) {
-  // Generate files in temporary folder first
-  const filePaths = await generateFilesInTempDir(tempDir, systemKey, application, dataSources);
+async function processDownloadedSystem(systemKey, manifest, splitOptions = {}) {
+  const { application, dataSources, version } = manifest;
+  const finalPath = getIntegrationPath(systemKey);
 
-  // Determine final destination (integration folder)
-  const { appPath } = await detectAppType(systemKey);
-  const finalPath = appPath || path.join(process.cwd(), 'integration', systemKey);
+  logger.log(chalk.blue(`📁 Creating directory: ${finalPath}`));
+  await fs.mkdir(finalPath, { recursive: true });
 
-  // Move files from temp to final location
-  await moveFilesToFinalLocation(tempDir, finalPath, systemKey, filePaths);
+  const deployJson = buildDeployJsonFromManifest(application, dataSources, version);
+  const deployJsonPath = path.join(finalPath, `${systemKey}-deploy.json`);
+  await fs.writeFile(deployJsonPath, JSON.stringify(deployJson, null, 2), 'utf8');
+  logger.log(chalk.green(`✓ Created: ${path.relative(process.cwd(), deployJsonPath)}`));
+
+  logger.log(chalk.blue('📂 Splitting deploy JSON into component files...'));
+  const splitResult = await generator.splitDeployJson(deployJsonPath, finalPath, splitOptions);
+  await applyRetemplateToSystemFile(systemKey, splitResult.systemFile);
 
   try {
     const secretsEnsure = require('../core/secrets-ensure');
@@ -397,9 +380,6 @@ async function processDownloadedSystem(systemKey, application, dataSources, temp
     if (err.code !== 'ENOENT') logger.warn(`Could not ensure integration placeholder secrets: ${err.message}`);
   }
 
-  // Clean up temporary folder
-  await fs.rm(tempDir, { recursive: true, force: true });
-
   return finalPath;
 }
 
@@ -422,50 +402,52 @@ function displayDownloadSuccess(systemKey, finalPath, datasourceCount) {
  * @function downloadExternalSystem
  * @param {string} systemKey - System key or ID
  * @param {Object} options - Download options
+ * @param {string} [options.format] - Output format: 'yaml' (default) or 'json' (runs convert after split)
  * @param {string} [options.environment] - Environment (dev, tst, pro)
  * @param {string} [options.controller] - Controller URL
  * @param {boolean} [options.dryRun] - Show what would be downloaded without actually downloading
+ * @param {boolean} [options.force] - Overwrite existing README.md without prompting
  * @returns {Promise<void>} Resolves when download completes
  * @throws {Error} If download fails
  */
+async function runConvertToJsonIfRequested(systemKey) {
+  const { runConvert } = require('../commands/convert');
+  try {
+    await runConvert(systemKey, { format: 'json', force: true });
+    logger.log(chalk.green('✓ Converted component files to JSON'));
+  } catch (convertErr) {
+    throw new Error(`Download succeeded but convert to JSON failed: ${convertErr.message}`);
+  }
+}
+
 async function downloadExternalSystem(systemKey, options = {}) {
   validateSystemKeyFormat(systemKey);
 
+  const format = (options.format || 'yaml').toLowerCase();
+
   try {
     logger.log(chalk.blue(`\n📥 Downloading external system: ${systemKey}`));
 
-    // Get authentication and dataplane URL
     const config = await getConfig();
     const { authConfig, dataplaneUrl } = await setupAuthenticationAndDataplane(systemKey, options, config);
 
-    // Handle dry run
     if (options.dryRun) {
       handleDryRun(systemKey, dataplaneUrl);
       return;
     }
 
-    // Download system configuration
-    const { application, dataSources } = await downloadSystemConfiguration(dataplaneUrl, systemKey, authConfig);
-
-    // Validate downloaded data
-    validateAndLogDownloadedData(application, dataSources);
-
-    // Create temporary folder for validation
-    const tempDir = path.join(os.tmpdir(), `aifabrix-download-${systemKey}-${Date.now()}`);
-    await fs.mkdir(tempDir, { recursive: true });
-
-    try {
-      const finalPath = await processDownloadedSystem(systemKey, application, dataSources, tempDir);
-      displayDownloadSuccess(systemKey, finalPath, dataSources.length);
-    } catch (error) {
-      // Clean up temporary folder on error
-      try {
-        await fs.rm(tempDir, { recursive: true, force: true });
-      } catch {
-        // Ignore cleanup errors
-      }
-      throw error;
+    const manifest = await downloadFullManifest(dataplaneUrl, systemKey, authConfig);
+    validateAndLogDownloadedData(manifest.application, manifest.dataSources);
+
+    const finalPath = getIntegrationPath(systemKey);
+    const splitOptions = await resolveDownloadSplitOptions(finalPath, options);
+    await processDownloadedSystem(systemKey, manifest, splitOptions);
+
+    if (format === 'json') {
+      await runConvertToJsonIfRequested(systemKey);
     }
+
+    displayDownloadSuccess(systemKey, finalPath, manifest.dataSources.length);
   } catch (error) {
     throw new Error(`Failed to download external system: ${error.message}`);
   }
@@ -474,9 +456,5 @@ async function downloadExternalSystem(systemKey, options = {}) {
 module.exports = {
   downloadExternalSystem,
   validateSystemType,
-  validateDownloadedData,
-  generateVariablesYaml,
-  generateReadme,
-  generateEnvTemplate,
-  handlePartialDownload
+  validateDownloadedData
 };