@aifabrix/builder 2.46.0 → 2.47.1

package/lib/datasource/orphan-profiles-validator.js

@@ -0,0 +1,98 @@
+/**
+ * Validate exposed.profiles keys against capabilities (346 §13 — no orphan profiles).
+ *
+ * @fileoverview Orphan exposed profile validation for datasource manifests
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { listCapabilityKeys } = require('./capability/capability-manifest');
+
+const FORBIDDEN_ORPHAN_PROFILE_KEYS = new Set(['minimal', 'analytics', 'metadata']);
+
+/**
+ * @param {string} name
+ * @returns {string}
+ */
+function normalizeProfileKey(name) {
+  return String(name).trim().toLowerCase();
+}
+
+/**
+ * @param {object} parsed
+ * @returns {Set<string>}
+ */
+function operationKeysFromConfig(parsed) {
+  const keys = new Set();
+  const openapiOps = parsed?.openapi?.operations;
+  if (openapiOps && typeof openapiOps === 'object') {
+    Object.keys(openapiOps).forEach((k) => keys.add(normalizeProfileKey(k)));
+  }
+  const cipOps = parsed?.execution?.cip?.operations;
+  if (cipOps && typeof cipOps === 'object') {
+    Object.keys(cipOps).forEach((k) => keys.add(normalizeProfileKey(k)));
+  }
+  return keys;
+}
+
+/**
+ * @param {object} parsed
+ * @returns {Set<string>}
+ */
+function allowedExposedProfileKeys(parsed) {
+  const allowed = new Set(['default']);
+  const capabilityKeys = listCapabilityKeys(parsed?.capabilities).map(normalizeProfileKey);
+  if (capabilityKeys.length > 0) {
+    capabilityKeys.forEach((k) => allowed.add(k));
+  } else {
+    operationKeysFromConfig(parsed).forEach((k) => allowed.add(k));
+  }
+  return allowed;
+}
+
+/**
+ * @param {object} parsed - Parsed datasource manifest
+ * @returns {string[]} Error messages; empty when valid
+ */
+function validateOrphanExposedProfiles(parsed) {
+  const errors = [];
+  const profiles = parsed?.exposed?.profiles;
+  if (!profiles || typeof profiles !== 'object' || Array.isArray(profiles)) {
+    return errors;
+  }
+
+  const allowed = allowedExposedProfileKeys(parsed);
+  const orphans = Object.keys(profiles).filter((name) => {
+    const normalized = normalizeProfileKey(name);
+    return FORBIDDEN_ORPHAN_PROFILE_KEYS.has(normalized) || !allowed.has(normalized);
+  });
+
+  if (orphans.length === 0) {
+    return errors;
+  }
+
+  const decorative = orphans.filter((name) =>
+    FORBIDDEN_ORPHAN_PROFILE_KEYS.has(normalizeProfileKey(name))
+  );
+  if (decorative.length > 0) {
+    errors.push(
+      `exposed.profiles: orphan exposed profiles are forbidden: ${decorative.join(', ')}. ` +
+        'Use exposed.profiles.default and one profile per capabilities[].key only.'
+    );
+    return errors;
+  }
+
+  errors.push(
+    `exposed.profiles: orphan profile keys must match capabilities[].key or default: ${orphans.join(', ')}. ` +
+      `Allowed profile keys: ${[...allowed].sort().join(', ')}.`
+  );
+  return errors;
+}
+
+module.exports = {
+  FORBIDDEN_ORPHAN_PROFILE_KEYS,
+  allowedExposedProfileKeys,
+  validateOrphanExposedProfiles
+};

package/lib/datasource/validate.js

@@ -10,6 +10,7 @@
 
 const path = require('path');
 const fs = require('fs');
+const { loadConfigFile } = require('../utils/config-format');
 const pathsUtil = require('../utils/paths');
 const {
   datasourceIdentifierMatchesFile,
@@ -22,7 +23,16 @@ const { validateFieldReferences } = require('./field-reference-validator');
 const { validateAbac } = require('./abac-validator');
 const { buildDatasourceValidateSummary } = require('./datasource-validate-summary');
 
-const EXCLUDE_JSON_NAMES = new Set(['application.json', 'rbac.json', 'package.json', 'package-lock.json']);
+const DATASOURCE_CONFIG_EXTENSIONS = new Set(['.json', '.yaml', '.yml']);
+const EXCLUDE_CONFIG_NAMES = new Set([
+  'application.json',
+  'application.yaml',
+  'application.yml',
+  'rbac.json',
+  'rbac.yaml',
+  'package.json',
+  'package-lock.json'
+]);
 
 /**
  * Resolve identifier to an existing file path, or null.
@@ -64,11 +74,14 @@ function pickIntegrationAppForDatasourceKey(datasourceKey, appNames) {
  * @param {string} fileName
  * @returns {boolean}
  */
-function shouldScanJsonDatasourceCandidate(fileName) {
+function shouldScanDatasourceConfigCandidate(fileName) {
   const lower = fileName.toLowerCase();
-  if (!lower.endsWith('.json')) return false;
-  if (EXCLUDE_JSON_NAMES.has(fileName)) return false;
+  const ext = path.extname(lower);
+  if (!DATASOURCE_CONFIG_EXTENSIONS.has(ext)) return false;
+  if (EXCLUDE_CONFIG_NAMES.has(fileName)) return false;
   if (lower.endsWith('-deploy.json') || lower.endsWith('-system.json')) return false;
+  if (lower.endsWith('-deploy.yaml') || lower.endsWith('-deploy.yml')) return false;
+  if (lower.endsWith('-system.yaml') || lower.endsWith('-system.yml')) return false;
   return true;
 }
 
@@ -86,13 +99,13 @@ function isDatasourceNamedJson(fileName) {
  * @param {string} datasourceKey
  * @returns {string[]}
  */
-function collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey) {
+function collectConfigPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey) {
   const hits = [];
   for (const name of scanOrder) {
     const full = path.join(dir, name);
     let parsed;
     try {
-      parsed = JSON.parse(fs.readFileSync(full, 'utf8'));
+      parsed = loadConfigFile(full);
     } catch {
       continue;
     }
@@ -115,14 +128,14 @@ function resolvePathFromIntegrationFolderScan(appName, datasourceKey) {
     throw new Error(`Integration folder not found: ${dir}`);
   }
   const entries = fs.readdirSync(dir);
-  const jsonFiles = entries.filter(shouldScanJsonDatasourceCandidate);
-  const prefer = jsonFiles.filter(isDatasourceNamedJson);
-  const scanOrder = [...prefer, ...jsonFiles.filter((f) => !prefer.includes(f))];
-  const hits = collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey);
+  const configFiles = entries.filter(shouldScanDatasourceConfigCandidate);
+  const prefer = configFiles.filter(isDatasourceNamedJson);
+  const scanOrder = [...prefer, ...configFiles.filter((f) => !prefer.includes(f))];
+  const hits = collectConfigPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey);
   if (hits.length === 0) {
     throw new Error(
-      `No datasource JSON matching "${datasourceKey}" under integration/${appName}/. ` +
-        `Checked: ${scanOrder.length ? scanOrder.join(', ') : '(no JSON candidates)'}`
+      `No datasource config matching "${datasourceKey}" under integration/${appName}/. ` +
+        `Checked: ${scanOrder.length ? scanOrder.join(', ') : '(no JSON/YAML candidates)'}`
     );
   }
   if (hits.length > 1) {
@@ -167,7 +180,8 @@ function resolveValidateInputPath(identifier) {
   }
   const asFile = tryResolveAsExistingFile(trimmed);
   if (asFile) return asFile;
-  if (trimmed.toLowerCase().endsWith('.json')) {
+  const ext = path.extname(trimmed).toLowerCase();
+  if (DATASOURCE_CONFIG_EXTENSIONS.has(ext)) {
     throw new Error(`File not found: ${trimmed}`);
   }
   return resolvePathFromIntegrationDatasourceKey(trimmed);
@@ -187,6 +201,18 @@ function resolveValidateInputPath(identifier) {
  * const result = await validateDatasourceFile('./hubspot-deal.json');
  * // Returns: { valid: true, errors: [], warnings: [], resolvedPath: '...' }
  */
+function buildParseFailureResult(filePath, error) {
+  const message =
+    error && error.message ? String(error.message) : 'Invalid config file';
+  return {
+    valid: false,
+    errors: [message],
+    warnings: [],
+    resolvedPath: filePath,
+    summary: null
+  };
+}
+
 async function validateDatasourceFile(filePathOrKey) {
   if (!filePathOrKey || typeof filePathOrKey !== 'string') {
     throw new Error('File path is required and must be a string');
@@ -194,19 +220,11 @@ async function validateDatasourceFile(filePathOrKey) {
 
   const filePath = resolveValidateInputPath(filePathOrKey.trim());
 
-  const content = fs.readFileSync(filePath, 'utf8');
   let parsed;
-
   try {
-    parsed = JSON.parse(content);
+    parsed = loadConfigFile(filePath);
   } catch (error) {
-    return {
-      valid: false,
-      errors: [`Invalid JSON syntax: ${error.message}`],
-      warnings: [],
-      resolvedPath: filePath,
-      summary: null
-    };
+    return buildParseFailureResult(filePath, error);
   }
 
   const summary = buildDatasourceValidateSummary(parsed);

package/lib/external-system/download-helpers.js

@@ -8,8 +8,90 @@
  * @version 2.0.0
  */
 
+const {
+  formatSuccessLine,
+  formatSuccessParagraph,
+  sectionTitle,
+  headerKeyValue
+} = require('../utils/cli-test-layout-chalk');
+const logger = require('../utils/logger');
 const { generateExternalReadmeContent } = require('../utils/external-readme');
 
+/**
+ * Validates system type from downloaded application
+ * @param {Object} application - External system configuration
+ * @returns {string} System type (openapi, mcp, custom)
+ */
+function validateSystemType(application) {
+  if (!application || typeof application !== 'object') {
+    throw new Error('Application configuration is required');
+  }
+  const validTypes = ['openapi', 'mcp', 'custom'];
+  const systemType = application.type;
+  if (!systemType || !validTypes.includes(systemType)) {
+    throw new Error(`Invalid system type: ${systemType}. Must be one of: ${validTypes.join(', ')}`);
+  }
+  return systemType;
+}
+
+function validateApplicationConfig(application) {
+  if (!application || typeof application !== 'object') {
+    throw new Error('Application configuration is required');
+  }
+  if (!application.key || typeof application.key !== 'string') {
+    throw new Error('Application key is required');
+  }
+}
+
+function validateDatasourceConfig(datasource, applicationKey) {
+  if (!datasource.key || typeof datasource.key !== 'string') {
+    throw new Error('Datasource key is required for all datasources');
+  }
+  if (!datasource.systemKey || typeof datasource.systemKey !== 'string') {
+    throw new Error('Datasource systemKey is required for all datasources');
+  }
+  if (datasource.systemKey !== applicationKey) {
+    throw new Error(
+      `Datasource systemKey (${datasource.systemKey}) does not match application key (${applicationKey})`
+    );
+  }
+}
+
+function validateDownloadedData(application, dataSources) {
+  validateApplicationConfig(application);
+  if (!Array.isArray(dataSources)) {
+    throw new Error('DataSources must be an array');
+  }
+  for (const datasource of dataSources) {
+    validateDatasourceConfig(datasource, application.key);
+  }
+}
+
+function logDownloadCommandHeader(systemKey) {
+  logger.log('');
+  logger.log(sectionTitle('Download'));
+  logger.log(headerKeyValue('System:', systemKey));
+}
+
+function displayDownloadSuccess(systemKey, finalPath, datasourceCount) {
+  logger.log(
+    formatSuccessParagraph(
+      `Downloaded ${systemKey} (${datasourceCount} datasource${datasourceCount === 1 ? '' : 's'})`
+    )
+  );
+  logger.log(headerKeyValue('Location:', finalPath));
+}
+
+async function runConvertToJsonIfRequested(systemKey) {
+  const { runConvert } = require('../commands/convert');
+  try {
+    await runConvert(systemKey, { format: 'json', force: true });
+    logger.log(formatSuccessLine('Converted component files to JSON'));
+  } catch (convertErr) {
+    throw new Error(`Download succeeded but convert to JSON failed: ${convertErr.message}`);
+  }
+}
+
 /**
  * Generates application.yaml content for downloaded system
  * @param {string} systemKey - System key
@@ -90,6 +172,11 @@ function generateReadme(systemKey, application, dataSources) {
 }
 
 module.exports = {
+  validateSystemType,
+  validateDownloadedData,
+  logDownloadCommandHeader,
+  displayDownloadSuccess,
+  runConvertToJsonIfRequested,
   generateVariablesYaml,
   generateReadme
 };

package/lib/external-system/download.js

@@ -1,8 +1,5 @@
 const {
   formatSuccessLine,
-  formatSuccessParagraph,
-  sectionTitle,
-  headerKeyValue,
   metadata,
   formatProgress,
   formatWarningLine
@@ -36,83 +33,18 @@ const { getExternalSystemConfig } = require('../api/external-systems.api');
 const { getDeploymentAuth, requireBearerForDataplanePipeline } = require('../utils/token-manager');
 const { getConfig } = require('../core/config');
 const { getIntegrationPath } = require('../utils/paths');
-const { retemplateConfigurationForDownload } = require('../utils/configuration-env-resolver');
+const { retemplateConfigurationForDownload, retemplateAllVariableDefinitions } = require('../utils/configuration-env-resolver');
+const { syncRuntimeConfigurationFromDataplane } = require('../utils/runtime-configuration-download');
 const logger = require('../utils/logger');
 const { resolveControllerUrl } = require('../utils/controller-url');
 const generator = require('../generator');
-
-/**
- * Validates system type from downloaded application
- * @param {Object} application - External system configuration
- * @returns {string} System type (openapi, mcp, custom)
- * @throws {Error} If system type is invalid
- */
-function validateSystemType(application) {
-  if (!application || typeof application !== 'object') {
-    throw new Error('Application configuration is required');
-  }
-
-  const validTypes = ['openapi', 'mcp', 'custom'];
-  const systemType = application.type;
-
-  if (!systemType || !validTypes.includes(systemType)) {
-    throw new Error(`Invalid system type: ${systemType}. Must be one of: ${validTypes.join(', ')}`);
-  }
-
-  return systemType;
-}
-
-/**
- * Validates downloaded data structure before writing files
- * @param {Object} application - External system configuration
- * @param {Array} dataSources - Array of datasource configurations
- * @throws {Error} If validation fails
- */
-/**
- * Validates application configuration
- * @function validateApplicationConfig
- * @param {Object} application - Application configuration
- * @throws {Error} If validation fails
- */
-function validateApplicationConfig(application) {
-  if (!application || typeof application !== 'object') {
-    throw new Error('Application configuration is required');
-  }
-  if (!application.key || typeof application.key !== 'string') {
-    throw new Error('Application key is required');
-  }
-}
-
-/**
- * Validates datasource configuration
- * @function validateDatasourceConfig
- * @param {Object} datasource - Datasource configuration
- * @param {string} applicationKey - Application key
- * @throws {Error} If validation fails
- */
-function validateDatasourceConfig(datasource, applicationKey) {
-  if (!datasource.key || typeof datasource.key !== 'string') {
-    throw new Error('Datasource key is required for all datasources');
-  }
-  if (!datasource.systemKey || typeof datasource.systemKey !== 'string') {
-    throw new Error('Datasource systemKey is required for all datasources');
-  }
-  if (datasource.systemKey !== applicationKey) {
-    throw new Error(`Datasource systemKey (${datasource.systemKey}) does not match application key (${applicationKey})`);
-  }
-}
-
-function validateDownloadedData(application, dataSources) {
-  validateApplicationConfig(application);
-
-  if (!Array.isArray(dataSources)) {
-    throw new Error('DataSources must be an array');
-  }
-
-  for (const datasource of dataSources) {
-    validateDatasourceConfig(datasource, application.key);
-  }
-}
+const {
+  validateSystemType,
+  validateDownloadedData,
+  logDownloadCommandHeader,
+  displayDownloadSuccess,
+  runConvertToJsonIfRequested
+} = require('./download-helpers');
 
 /**
  * Setup authentication and get dataplane URL
@@ -354,10 +286,9 @@ async function applyRetemplateToSystemFile(systemKey, systemFilePath) {
   const systemContent = await fs.readFile(systemFilePath, 'utf8');
   const systemObj = yaml.load(systemContent);
   if (!Array.isArray(systemObj?.configuration)) return;
-  const applied = await retemplateConfigurationForDownload(systemKey, systemObj.configuration);
-  if (applied) {
-    await fs.writeFile(systemFilePath, yaml.dump(systemObj, { indent: 2, lineWidth: -1 }), 'utf8');
-  }
+  retemplateAllVariableDefinitions(systemObj.configuration);
+  await retemplateConfigurationForDownload(systemKey, systemObj.configuration);
+  await fs.writeFile(systemFilePath, yaml.dump(systemObj, { indent: 2, lineWidth: -1 }), 'utf8');
 }
 
 /**
@@ -397,43 +328,54 @@ async function processDownloadedSystem(systemKey, manifest, splitOptions = {}) {
 }
 
 /**
- * Display download success message
- * @param {string} systemKey - System key
- * @param {string} finalPath - Final destination path
- * @param {number} datasourceCount - Number of datasources
- */
-function displayDownloadSuccess(systemKey, finalPath, datasourceCount) {
-  logger.log(
-    formatSuccessParagraph(
-      `Downloaded ${systemKey} (${datasourceCount} datasource${datasourceCount === 1 ? '' : 's'})`
-    )
-  );
-  logger.log(headerKeyValue('Location:', finalPath));
-}
-
-/**
- * After YAML split, convert integration files to JSON when --format json.
- * @async
- * @param {string} systemKey
+ * Post-split download steps: governance packs, optional JSON convert, runtime config sync.
+ * @param {Object} params
+ * @param {string} params.systemKey
+ * @param {Object} params.manifest
+ * @param {string} params.finalPath
+ * @param {string} params.dataplaneUrl
+ * @param {Object} params.authConfig
+ * @param {string} params.format
  * @returns {Promise<void>}
  */
-async function runConvertToJsonIfRequested(systemKey) {
-  const { runConvert } = require('../commands/convert');
+async function finalizeDownloadArtifacts({
+  systemKey,
+  manifest,
+  finalPath,
+  dataplaneUrl,
+  authConfig,
+  format
+}) {
+  const { downloadGovernanceScenarioPacksFromManifest } = require('../lifecycle/scenario-upload');
+  await downloadGovernanceScenarioPacksFromManifest(
+    systemKey,
+    manifest,
+    dataplaneUrl,
+    authConfig
+  );
+  if (format === 'json') {
+    await runConvertToJsonIfRequested(systemKey);
+  }
   try {
-    await runConvert(systemKey, { format: 'json', force: true });
-    logger.log(formatSuccessLine('Converted component files to JSON'));
-  } catch (convertErr) {
-    throw new Error(`Download succeeded but convert to JSON failed: ${convertErr.message}`);
+    const configSync = await syncRuntimeConfigurationFromDataplane({
+      dataplaneUrl,
+      authConfig,
+      systemKey,
+      integrationPath: finalPath,
+      dataSources: manifest.dataSources
+    });
+    if (configSync.merged > 0) {
+      const nameList = configSync.names.length ? ` (${configSync.names.join(', ')})` : '';
+      logger.log(
+        formatSuccessLine(
+          `Merged ${configSync.merged} runtime configuration variable(s) into env.template${nameList}.`
+        )
+      );
+    }
+  } catch (syncErr) {
+    logger.log(formatWarningLine(`Runtime configuration sync skipped: ${syncErr.message}`));
   }
-}
-
-/**
- * @param {string} systemKey
- */
-function logDownloadCommandHeader(systemKey) {
-  logger.log('');
-  logger.log(sectionTitle('Download'));
-  logger.log(headerKeyValue('System:', systemKey));
+  displayDownloadSuccess(systemKey, finalPath, manifest.dataSources.length);
 }
 
 /**
@@ -468,17 +410,14 @@ async function downloadExternalSystem(systemKey, options = {}) {
       manifest,
       await resolveDownloadSplitOptions(finalPath, options)
     );
-    const { downloadGovernanceScenarioPacksFromManifest } = require('../lifecycle/scenario-upload');
-    await downloadGovernanceScenarioPacksFromManifest(
+    await finalizeDownloadArtifacts({
       systemKey,
       manifest,
+      finalPath,
       dataplaneUrl,
-      authConfig
-    );
-    if (format === 'json') {
-      await runConvertToJsonIfRequested(systemKey);
-    }
-    displayDownloadSuccess(systemKey, finalPath, manifest.dataSources.length);
+      authConfig,
+      format
+    });
   } catch (error) {
     throw new Error(`Failed to download external system: ${error.message}`);
   }

package/lib/external-system/generator.js

@@ -16,70 +16,12 @@ const handlebars = require('handlebars');
 const logger = require('../utils/logger');
 const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
 const { loadConfigFile, writeConfigFile } = require('../utils/config-format');
-const { getKvPathSegmentForSecurityKey } = require('../utils/credential-secrets-env');
+const { buildAuthenticationFromMethod } = require('../utils/authentication-shape');
 
 // Register Handlebars helper for equality check
 handlebars.registerHelper('eq', (a, b) => a === b);
 handlebars.registerHelper('json', (obj) => JSON.stringify(obj));
 
-/**
- * Build authentication object per schema authenticationVariablesByMethod.
- * Security values use canonical kv://<systemKey>/<segment> paths (segment from getKvPathSegmentForSecurityKey).
- * @param {string} systemKey - External system key
- * @param {string} authType - Auth method (oauth2, aad, apikey, bearerToken, basic, queryParam, oidc, hmac, none)
- * @returns {{ method: string, variables: Object, security: Object }} Authentication object
- */
-function buildAuthenticationFromMethod(systemKey, authType) {
-  const kvPath = (securityKey) => {
-    const segment = getKvPathSegmentForSecurityKey(securityKey);
-    return segment ? `kv://${systemKey}/${segment}` : null;
-  };
-  const method = authType || 'apikey';
-  const base = 'https://api.example.com';
-
-  const authMap = {
-    oauth2: {
-      variables: { baseUrl: base, tokenUrl: `${base}/oauth/token`, authorizationUrl: `${base}/oauth/authorize` },
-      security: { clientId: kvPath('clientId'), clientSecret: kvPath('clientSecret') }
-    },
-    aad: {
-      variables: { baseUrl: base, tokenUrl: 'https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token', tenantId: '{tenant-id}' },
-      security: { clientId: kvPath('clientId'), clientSecret: kvPath('clientSecret') }
-    },
-    apikey: {
-      variables: { baseUrl: base, headerName: 'X-API-Key' },
-      security: { apiKey: kvPath('apiKey') }
-    },
-    bearerToken: {
-      variables: { baseUrl: base, headerName: 'Authorization', prefix: 'Bearer' },
-      security: { token: kvPath('token') }
-    },
-    basic: {
-      variables: { baseUrl: base },
-      security: { username: kvPath('username'), password: kvPath('password') }
-    },
-    queryParam: {
-      variables: { baseUrl: base, paramName: 'api_key' },
-      security: { paramValue: kvPath('paramValue') }
-    },
-    oidc: {
-      variables: { openIdConfigUrl: 'https://example.com/.well-known/openid-configuration', clientId: 'app-id' },
-      security: {}
-    },
-    hmac: {
-      variables: { baseUrl: base, algorithm: 'sha256', signatureHeader: 'X-Signature' },
-      security: { signingSecret: kvPath('signingSecret') }
-    },
-    none: {
-      variables: {},
-      security: {}
-    }
-  };
-
-  const auth = authMap[method] || authMap.apikey;
-  return { method, variables: auth.variables, security: auth.security };
-}
-
 /** Target extension for format */
 const FORMAT_EXT = { yaml: '.yaml', json: '.json' };