@aifabrix/builder 2.46.0 → 2.47.0

package/lib/external-system/download.js

@@ -1,8 +1,5 @@
 const {
   formatSuccessLine,
-  formatSuccessParagraph,
-  sectionTitle,
-  headerKeyValue,
   metadata,
   formatProgress,
   formatWarningLine
@@ -36,83 +33,18 @@ const { getExternalSystemConfig } = require('../api/external-systems.api');
 const { getDeploymentAuth, requireBearerForDataplanePipeline } = require('../utils/token-manager');
 const { getConfig } = require('../core/config');
 const { getIntegrationPath } = require('../utils/paths');
-const { retemplateConfigurationForDownload } = require('../utils/configuration-env-resolver');
+const { retemplateConfigurationForDownload, retemplateAllVariableDefinitions } = require('../utils/configuration-env-resolver');
+const { syncRuntimeConfigurationFromDataplane } = require('../utils/runtime-configuration-download');
 const logger = require('../utils/logger');
 const { resolveControllerUrl } = require('../utils/controller-url');
 const generator = require('../generator');
-
-/**
- * Validates system type from downloaded application
- * @param {Object} application - External system configuration
- * @returns {string} System type (openapi, mcp, custom)
- * @throws {Error} If system type is invalid
- */
-function validateSystemType(application) {
-  if (!application || typeof application !== 'object') {
-    throw new Error('Application configuration is required');
-  }
-
-  const validTypes = ['openapi', 'mcp', 'custom'];
-  const systemType = application.type;
-
-  if (!systemType || !validTypes.includes(systemType)) {
-    throw new Error(`Invalid system type: ${systemType}. Must be one of: ${validTypes.join(', ')}`);
-  }
-
-  return systemType;
-}
-
-/**
- * Validates downloaded data structure before writing files
- * @param {Object} application - External system configuration
- * @param {Array} dataSources - Array of datasource configurations
- * @throws {Error} If validation fails
- */
-/**
- * Validates application configuration
- * @function validateApplicationConfig
- * @param {Object} application - Application configuration
- * @throws {Error} If validation fails
- */
-function validateApplicationConfig(application) {
-  if (!application || typeof application !== 'object') {
-    throw new Error('Application configuration is required');
-  }
-  if (!application.key || typeof application.key !== 'string') {
-    throw new Error('Application key is required');
-  }
-}
-
-/**
- * Validates datasource configuration
- * @function validateDatasourceConfig
- * @param {Object} datasource - Datasource configuration
- * @param {string} applicationKey - Application key
- * @throws {Error} If validation fails
- */
-function validateDatasourceConfig(datasource, applicationKey) {
-  if (!datasource.key || typeof datasource.key !== 'string') {
-    throw new Error('Datasource key is required for all datasources');
-  }
-  if (!datasource.systemKey || typeof datasource.systemKey !== 'string') {
-    throw new Error('Datasource systemKey is required for all datasources');
-  }
-  if (datasource.systemKey !== applicationKey) {
-    throw new Error(`Datasource systemKey (${datasource.systemKey}) does not match application key (${applicationKey})`);
-  }
-}
-
-function validateDownloadedData(application, dataSources) {
-  validateApplicationConfig(application);
-
-  if (!Array.isArray(dataSources)) {
-    throw new Error('DataSources must be an array');
-  }
-
-  for (const datasource of dataSources) {
-    validateDatasourceConfig(datasource, application.key);
-  }
-}
+const {
+  validateSystemType,
+  validateDownloadedData,
+  logDownloadCommandHeader,
+  displayDownloadSuccess,
+  runConvertToJsonIfRequested
+} = require('./download-helpers');
 
 /**
  * Setup authentication and get dataplane URL
@@ -354,10 +286,9 @@ async function applyRetemplateToSystemFile(systemKey, systemFilePath) {
   const systemContent = await fs.readFile(systemFilePath, 'utf8');
   const systemObj = yaml.load(systemContent);
   if (!Array.isArray(systemObj?.configuration)) return;
-  const applied = await retemplateConfigurationForDownload(systemKey, systemObj.configuration);
-  if (applied) {
-    await fs.writeFile(systemFilePath, yaml.dump(systemObj, { indent: 2, lineWidth: -1 }), 'utf8');
-  }
+  retemplateAllVariableDefinitions(systemObj.configuration);
+  await retemplateConfigurationForDownload(systemKey, systemObj.configuration);
+  await fs.writeFile(systemFilePath, yaml.dump(systemObj, { indent: 2, lineWidth: -1 }), 'utf8');
 }
 
 /**
@@ -397,43 +328,54 @@ async function processDownloadedSystem(systemKey, manifest, splitOptions = {}) {
 }
 
 /**
- * Display download success message
- * @param {string} systemKey - System key
- * @param {string} finalPath - Final destination path
- * @param {number} datasourceCount - Number of datasources
- */
-function displayDownloadSuccess(systemKey, finalPath, datasourceCount) {
-  logger.log(
-    formatSuccessParagraph(
-      `Downloaded ${systemKey} (${datasourceCount} datasource${datasourceCount === 1 ? '' : 's'})`
-    )
-  );
-  logger.log(headerKeyValue('Location:', finalPath));
-}
-
-/**
- * After YAML split, convert integration files to JSON when --format json.
- * @async
- * @param {string} systemKey
+ * Post-split download steps: governance packs, optional JSON convert, runtime config sync.
+ * @param {Object} params
+ * @param {string} params.systemKey
+ * @param {Object} params.manifest
+ * @param {string} params.finalPath
+ * @param {string} params.dataplaneUrl
+ * @param {Object} params.authConfig
+ * @param {string} params.format
  * @returns {Promise<void>}
  */
-async function runConvertToJsonIfRequested(systemKey) {
-  const { runConvert } = require('../commands/convert');
+async function finalizeDownloadArtifacts({
+  systemKey,
+  manifest,
+  finalPath,
+  dataplaneUrl,
+  authConfig,
+  format
+}) {
+  const { downloadGovernanceScenarioPacksFromManifest } = require('../lifecycle/scenario-upload');
+  await downloadGovernanceScenarioPacksFromManifest(
+    systemKey,
+    manifest,
+    dataplaneUrl,
+    authConfig
+  );
+  if (format === 'json') {
+    await runConvertToJsonIfRequested(systemKey);
+  }
   try {
-    await runConvert(systemKey, { format: 'json', force: true });
-    logger.log(formatSuccessLine('Converted component files to JSON'));
-  } catch (convertErr) {
-    throw new Error(`Download succeeded but convert to JSON failed: ${convertErr.message}`);
+    const configSync = await syncRuntimeConfigurationFromDataplane({
+      dataplaneUrl,
+      authConfig,
+      systemKey,
+      integrationPath: finalPath,
+      dataSources: manifest.dataSources
+    });
+    if (configSync.merged > 0) {
+      const nameList = configSync.names.length ? ` (${configSync.names.join(', ')})` : '';
+      logger.log(
+        formatSuccessLine(
+          `Merged ${configSync.merged} runtime configuration variable(s) into env.template${nameList}.`
+        )
+      );
+    }
+  } catch (syncErr) {
+    logger.log(formatWarningLine(`Runtime configuration sync skipped: ${syncErr.message}`));
   }
-}
-
-/**
- * @param {string} systemKey
- */
-function logDownloadCommandHeader(systemKey) {
-  logger.log('');
-  logger.log(sectionTitle('Download'));
-  logger.log(headerKeyValue('System:', systemKey));
+  displayDownloadSuccess(systemKey, finalPath, manifest.dataSources.length);
 }
 
 /**
@@ -468,17 +410,14 @@ async function downloadExternalSystem(systemKey, options = {}) {
       manifest,
       await resolveDownloadSplitOptions(finalPath, options)
     );
-    const { downloadGovernanceScenarioPacksFromManifest } = require('../lifecycle/scenario-upload');
-    await downloadGovernanceScenarioPacksFromManifest(
+    await finalizeDownloadArtifacts({
       systemKey,
       manifest,
+      finalPath,
       dataplaneUrl,
-      authConfig
-    );
-    if (format === 'json') {
-      await runConvertToJsonIfRequested(systemKey);
-    }
-    displayDownloadSuccess(systemKey, finalPath, manifest.dataSources.length);
+      authConfig,
+      format
+    });
   } catch (error) {
     throw new Error(`Failed to download external system: ${error.message}`);
   }

package/lib/external-system/generator.js

@@ -16,70 +16,12 @@ const handlebars = require('handlebars');
 const logger = require('../utils/logger');
 const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
 const { loadConfigFile, writeConfigFile } = require('../utils/config-format');
-const { getKvPathSegmentForSecurityKey } = require('../utils/credential-secrets-env');
+const { buildAuthenticationFromMethod } = require('../utils/authentication-shape');
 
 // Register Handlebars helper for equality check
 handlebars.registerHelper('eq', (a, b) => a === b);
 handlebars.registerHelper('json', (obj) => JSON.stringify(obj));
 
-/**
- * Build authentication object per schema authenticationVariablesByMethod.
- * Security values use canonical kv://<systemKey>/<segment> paths (segment from getKvPathSegmentForSecurityKey).
- * @param {string} systemKey - External system key
- * @param {string} authType - Auth method (oauth2, aad, apikey, bearerToken, basic, queryParam, oidc, hmac, none)
- * @returns {{ method: string, variables: Object, security: Object }} Authentication object
- */
-function buildAuthenticationFromMethod(systemKey, authType) {
-  const kvPath = (securityKey) => {
-    const segment = getKvPathSegmentForSecurityKey(securityKey);
-    return segment ? `kv://${systemKey}/${segment}` : null;
-  };
-  const method = authType || 'apikey';
-  const base = 'https://api.example.com';
-
-  const authMap = {
-    oauth2: {
-      variables: { baseUrl: base, tokenUrl: `${base}/oauth/token`, authorizationUrl: `${base}/oauth/authorize` },
-      security: { clientId: kvPath('clientId'), clientSecret: kvPath('clientSecret') }
-    },
-    aad: {
-      variables: { baseUrl: base, tokenUrl: 'https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token', tenantId: '{tenant-id}' },
-      security: { clientId: kvPath('clientId'), clientSecret: kvPath('clientSecret') }
-    },
-    apikey: {
-      variables: { baseUrl: base, headerName: 'X-API-Key' },
-      security: { apiKey: kvPath('apiKey') }
-    },
-    bearerToken: {
-      variables: { baseUrl: base, headerName: 'Authorization', prefix: 'Bearer' },
-      security: { token: kvPath('token') }
-    },
-    basic: {
-      variables: { baseUrl: base },
-      security: { username: kvPath('username'), password: kvPath('password') }
-    },
-    queryParam: {
-      variables: { baseUrl: base, paramName: 'api_key' },
-      security: { paramValue: kvPath('paramValue') }
-    },
-    oidc: {
-      variables: { openIdConfigUrl: 'https://example.com/.well-known/openid-configuration', clientId: 'app-id' },
-      security: {}
-    },
-    hmac: {
-      variables: { baseUrl: base, algorithm: 'sha256', signatureHeader: 'X-Signature' },
-      security: { signingSecret: kvPath('signingSecret') }
-    },
-    none: {
-      variables: {},
-      security: {}
-    }
-  };
-
-  const auth = authMap[method] || authMap.apikey;
-  return { method, variables: auth.variables, security: auth.security };
-}
-
 /** Target extension for format */
 const FORMAT_EXT = { yaml: '.yaml', json: '.json' };
 

package/lib/external-system/test.js

@@ -7,7 +7,6 @@
  */
 /* eslint-disable max-lines,max-lines-per-function,max-statements,complexity,max-depth -- Integration test flow with system-level and per-datasource paths */
 
-const fs = require('fs').promises;
 const fsSync = require('fs');
 const path = require('path');
 const chalk = require('chalk');
@@ -69,15 +68,15 @@ async function loadVariablesYamlFile(variablesPath) {
 }
 
 /**
- * Loads a single JSON file
+ * Loads a single integration config file (JSON, YAML, or YML).
  * @async
- * @function loadJsonFile
- * @param {string} filePath - Path to JSON file
+ * @function loadIntegrationConfigFile
+ * @param {string} filePath - Path to config file
  * @param {string} fileName - File name for error messages
- * @returns {Promise<Object>} Parsed JSON data
- * @throws {Error} If file not found or invalid JSON
+ * @returns {Promise<Object>} Parsed config object
+ * @throws {Error} If file not found or invalid format
  */
-async function loadJsonFile(filePath, fileName) {
+async function loadIntegrationConfigFile(filePath, fileName) {
   if (!fsSync.existsSync(filePath)) {
     // Use "System file not found" for system files to match test expectations
     if (fileName.includes('system') || fileName.includes('deploy')) {
@@ -86,12 +85,8 @@ async function loadJsonFile(filePath, fileName) {
     throw new Error(`${fileName} not found: ${filePath}`);
   }
 
-  const content = await fs.readFile(filePath, 'utf8');
-  try {
-    return JSON.parse(content);
-  } catch (error) {
-    throw new Error(`Invalid JSON syntax in ${fileName}: ${error.message}`);
-  }
+  const { loadConfigFile } = require('../utils/config-format');
+  return loadConfigFile(filePath);
 }
 
 /**
@@ -121,7 +116,7 @@ async function loadSystemFiles(systemFiles, schemaBasePath, appPath) {
   const systemJsonFiles = [];
   for (const systemFile of systemFiles) {
     const systemPath = resolveFilePath(schemaBasePath, appPath, systemFile);
-    const systemJson = await loadJsonFile(systemPath, systemFile);
+    const systemJson = await loadIntegrationConfigFile(systemPath, systemFile);
     systemJsonFiles.push({ path: systemPath, data: systemJson });
   }
   return systemJsonFiles;
@@ -140,7 +135,7 @@ async function loadDatasourceFiles(datasourceFiles, schemaBasePath, appPath) {
   const datasourceJsonFiles = [];
   for (const datasourceFile of datasourceFiles) {
     const datasourcePath = resolveFilePath(schemaBasePath, appPath, datasourceFile);
-    const datasourceJson = await loadJsonFile(datasourcePath, datasourceFile);
+    const datasourceJson = await loadIntegrationConfigFile(datasourcePath, datasourceFile);
     datasourceJsonFiles.push({ path: datasourcePath, data: datasourceJson });
   }
   return datasourceJsonFiles;

package/lib/generator/builders.js

@@ -267,6 +267,15 @@ function validateBuildFields(build) {
   if (build.dockerfile && build.dockerfile.trim()) {
     buildConfig.dockerfile = build.dockerfile;
   }
+  if (build.worker === true) {
+    buildConfig.worker = true;
+  }
+  if (typeof build.imageRunWorker === 'string' && build.imageRunWorker.trim()) {
+    buildConfig.imageRunWorker = build.imageRunWorker.trim();
+  }
+  if (typeof build.imageRunBeat === 'string' && build.imageRunBeat.trim()) {
+    buildConfig.imageRunBeat = build.imageRunBeat.trim();
+  }
 
   return Object.keys(buildConfig).length > 0 ? buildConfig : null;
 }
@@ -390,6 +399,7 @@ module.exports = {
   buildOptionalFields,
   buildManifestStructure,
   filterConfigurationByRegistryMode,
-  sanitizeAuthType
+  sanitizeAuthType,
+  validateBuildFields
 };
 

package/lib/generator/external-controller-manifest.js

@@ -27,11 +27,14 @@ const { loadVariables, loadRbac } = require('./helpers');
  * @param {Object} systemJson - System JSON object (mutated in place)
  */
 function normalizeLegacyAuthMethod(systemJson) {
+  const { getAuthCredentialTemplate } = require('../utils/authentication-shape');
   const auth = systemJson && systemJson.authentication;
   if (!auth || typeof auth !== 'object') return;
-  const method = String(auth.method || '').trim().toLowerCase();
+  const credential = getAuthCredentialTemplate(auth);
+  if (!credential) return;
+  const method = String(credential.method || '').trim().toLowerCase();
   if (method === 'bearerkey') {
-    auth.method = 'bearerToken';
+    credential.method = 'bearerToken';
   }
 }
 

package/lib/generator/split.js

@@ -453,6 +453,8 @@ module.exports = {
   extractVariablesYaml,
   extractRbacYaml,
   parseImageReference,
-  generateReadmeFromDeployJson
+  generateReadmeFromDeployJson,
+  mergeEnvTemplateWithExisting,
+  buildEnvTemplateExpectedByKey
 };
 

package/lib/generator/wizard.js

@@ -32,34 +32,7 @@ function toKeySegment(str) {
   return sanitized || 'default';
 }
 
-/**
- * Align authentication.security kv:// namespaces and credentialKey with the final system key.
- * Dataplane normalizes this before respond; when appName overrides a spec-derived key (e.g.
- * OpenAPI title "Companies"), the builder still must rewrite nested auth so it matches env.template.
- *
- * @param {Object|null|undefined} authentication - authentication block from dataplane
- * @param {string} systemKey - Final external system key (integration app name)
- * @param {string} [authDisplayName] - Credential display name (typically title-cased app name)
- */
-function normalizeAuthenticationToSystemKey(authentication, systemKey, authDisplayName) {
-  if (!authentication || typeof authentication !== 'object') return;
-  authentication.credentialKey = `${systemKey}-cred`;
-  const security = authentication.security;
-  if (security && typeof security === 'object') {
-    for (const k of Object.keys(security)) {
-      const v = security[k];
-      if (typeof v === 'string' && v.startsWith('kv://')) {
-        const rest = v.slice(5);
-        const idx = rest.indexOf('/');
-        const suffix = idx >= 0 ? rest.slice(idx + 1) : '';
-        security[k] = suffix ? `kv://${systemKey}/${suffix}` : `kv://${systemKey}`;
-      }
-    }
-  }
-  if (authDisplayName) {
-    authentication.displayName = authDisplayName;
-  }
-}
+const { normalizeAuthenticationToSystemKey } = require('../utils/authentication-shape');
 
 /**
  * Generate files from dataplane-generated wizard configurations

package/lib/identity/fk-reference.js

@@ -0,0 +1,65 @@
+/**
+ * @fileoverview Parse ForeignKeyReference objects from controller identity APIs (plan 194)
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+/**
+ * @param {string|Object|null|undefined} ref
+ * @returns {string}
+ */
+function readFkId(ref) {
+  if (typeof ref === 'string') {
+    return ref;
+  }
+  return String(ref?.id || '').trim();
+}
+
+/**
+ * @param {string|Object|null|undefined} ref
+ * @returns {string}
+ */
+function readFkKey(ref) {
+  if (typeof ref === 'string') {
+    return ref;
+  }
+  return String(ref?.key || '').trim();
+}
+
+/**
+ * @param {string|Object|null|undefined} ref
+ * @returns {string}
+ */
+function readFkName(ref) {
+  if (typeof ref === 'string') {
+    return '';
+  }
+  return String(ref?.name || '').trim();
+}
+
+/**
+ * @param {string|Object|null|undefined} ref
+ * @param {string} [fallback]
+ * @returns {string}
+ */
+function formatFkLabel(ref, fallback = '—') {
+  const name = readFkName(ref);
+  if (name) {
+    return name;
+  }
+  const key = readFkKey(ref);
+  if (key) {
+    return key;
+  }
+  const id = readFkId(ref);
+  return id || fallback;
+}
+
+module.exports = {
+  readFkId,
+  readFkKey,
+  readFkName,
+  formatFkLabel
+};

package/lib/identity/identity-apply-core.js

@@ -6,46 +6,13 @@
 
 'use strict';
 
-const { getUser, listUsers } = require('../api/users.api');
-
-/**
- * @param {Object} response
- * @returns {any}
- */
-function unwrapControllerData(response) {
-  if (!response || response.success === false) {
-    return null;
-  }
-  return response?.data?.data ?? response?.data ?? response;
-}
-
-/**
- * @param {Object} response
- * @returns {string}
- */
-function apiErrorMessage(response) {
-  if (!response) {
-    return 'Request failed';
-  }
-  const err = response.error;
-  if (typeof err === 'string') {
-    return err;
-  }
-  if (err && typeof err === 'object') {
-    return err.formattedError || err.message || err.error || 'Request failed';
-  }
-  return response.formattedError || response.message || 'Request failed';
-}
-
-/**
- * @param {Object} response
- * @throws {Error}
- */
-function throwIfApiFailed(response) {
-  if (!response || response.success === false) {
-    throw new Error(apiErrorMessage(response));
-  }
-}
+const { lookupUserByIdOrEmail } = require('./resolve-user-id');
+const { buildSyncStatsFailureMessage } = require('./identity-sync-errors');
+const {
+  unwrapControllerData,
+  apiErrorMessage,
+  throwIfApiFailed
+} = require('./identity-response-helpers');
 
 /**
  * @param {Object|null|undefined} stats
@@ -55,11 +22,7 @@ function throwIfApiFailed(response) {
 function assertSyncStatsOk(stats, opts = {}) {
   const syncErrors = Number(stats?.errors ?? 0);
   if (syncErrors > 0) {
-    throw new Error(
-      `Dataplane full sync reported ${syncErrors} error(s). ` +
-        `usersProcessed=${stats?.usersProcessed ?? 0} groupsProcessed=${stats?.groupsProcessed ?? 0}. ` +
-        'Check miso-controller logs for the sync request.'
-    );
+    throw new Error(buildSyncStatsFailureMessage(stats));
   }
   const processed = Number(stats?.usersProcessed ?? 0);
   if (processed === 0 && !opts.allowEmptySync) {
@@ -79,24 +42,16 @@ function assertSyncStatsOk(stats, opts = {}) {
 async function findExistingUser(controllerUrl, authConfig, userSpec) {
   const csvId = String(userSpec.csvId || '').trim();
   if (csvId) {
-    const byId = await getUser(controllerUrl, authConfig, csvId);
-    const row = unwrapControllerData(byId);
-    if (row && row.id) {
-      return row;
+    const byId = await lookupUserByIdOrEmail(controllerUrl, authConfig, csvId);
+    if (byId?.id) {
+      return byId;
     }
   }
   const email = String(userSpec.email || '').trim();
-  const listed = await listUsers(controllerUrl, authConfig, {
-    search: email,
-    pageSize: 10,
-    page: 1
-  });
-  const items = unwrapControllerData(listed);
-  const list = Array.isArray(items) ? items : items?.data;
-  if (!Array.isArray(list)) {
+  if (!email) {
     return null;
   }
-  return list.find((u) => String(u.email || '').toLowerCase() === email.toLowerCase()) || null;
+  return lookupUserByIdOrEmail(controllerUrl, authConfig, email);
 }
 
 module.exports = {

package/lib/identity/identity-apply-phases.js

@@ -18,6 +18,7 @@ const {
   apiErrorMessage,
   findExistingUser
 } = require('./identity-apply-core');
+const { resolveIdentityUsername } = require('./username-default');
 
 /**
  * @param {string} msg
@@ -75,7 +76,10 @@ async function ensureUser(controllerUrl, authConfig, userSpec, dryRun) {
     firstName: userSpec.firstName,
     lastName: userSpec.lastName,
     displayName: userSpec.displayName,
-    username: userSpec.username,
+    username: resolveIdentityUsername({
+      username: userSpec.username,
+      email: userSpec.email
+    }),
     status: 'active'
   };
   Object.keys(body).forEach((k) => {