@aifabrix/builder 2.50.0 → 2.51.1

package/lib/external-system/deploy.js

@@ -30,6 +30,7 @@ const { displayValidationResults } = require('../validation/validate-display');
 const { syncDeployJsonFromSources } = require('./sync-deploy-manifest');
 const { runDeployGateOrchestrator } = require('../commands/deploy-gate-orchestrator');
 const { runPostPublishDataSync } = require('../commands/run-post-publish-data-sync');
+const { uploadDigitalWorkerIfNeeded } = require('./deploy-helpers');
 
 /**
  * Validates manifest on dataplane (dry-run) before controller deploy.
@@ -342,8 +343,20 @@ async function executeDeployAndDisplay(manifest, controllerUrl, environment, aut
  * @param {Object} options - Deployment options
  * @returns {Promise<Object>} Deployment configuration
  */
-async function prepareDeploymentConfig(appName, _options) {
+async function prepareDeploymentConfig(appName, options = {}) {
   const { resolveEnvironment } = require('../core/config');
+  if (options.authConfig && (options.authConfig.token || options.authConfig.clientId)) {
+    const environment = options.environment || (await resolveEnvironment());
+    let controllerUrl = options.controllerUrl || '';
+    if (!controllerUrl) {
+      controllerUrl = await resolveControllerUrl();
+    }
+    if (!controllerUrl) {
+      throw new Error('Controller URL is required. Run "aifabrix login" first.');
+    }
+    return { environment, controllerUrl, authConfig: options.authConfig };
+  }
+
   const environment = await resolveEnvironment();
   const controllerUrl = await resolveControllerUrl();
   const authConfig = await getDeploymentAuth(controllerUrl, environment, appName);
@@ -413,24 +426,26 @@ async function runExternalSystemDeployFlow(appName, options) {
   const compact = options.verbose !== true;
   const { appPath } = await detectAppType(appName);
   logOfflinePathWhenType(appPath);
-
   if (!compact) {
     logger.log(chalk.blue(`\n🚀 Deploying external system: ${appName}`));
   }
-
   await validateExternalSystemForDeploy(appName, options, compact);
-
   const gateOutcomes = await runDeployGateOrchestrator(appName, {
     ...options,
     quietDeployUi: compact
   });
-
   await syncDeployJsonFromSources(appName);
-
   const manifest = await generateControllerManifest(appName, options);
-
+  const digitalWorkerResult = await uploadDigitalWorkerIfNeeded(
+    appName,
+    options,
+    compact,
+    manifest
+  );
+  if (digitalWorkerResult) {
+    return digitalWorkerResult;
+  }
   const { environment, controllerUrl, authConfig } = await prepareDeploymentConfig(appName, options);
-
   let dataplaneUrl;
   try {
     dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig, {
@@ -439,9 +454,7 @@ async function runExternalSystemDeployFlow(appName, options) {
   } catch (err) {
     throw new Error(`Failed to resolve dataplane URL before deploy: ${err.message}`);
   }
-
   await validateDataplaneManifestForDeploy(manifest, authConfig, dataplaneUrl, compact);
-
   return executeDeployAndDisplay(manifest, controllerUrl, environment, authConfig, {
     ...options,
     gateOutcomes,

package/lib/external-system/download-helpers.js

@@ -11,12 +11,45 @@
 const {
   formatSuccessLine,
   formatSuccessParagraph,
+  formatWarningLine,
   sectionTitle,
   headerKeyValue
 } = require('../utils/cli-test-layout-chalk');
 const logger = require('../utils/logger');
 const { generateExternalReadmeContent } = require('../utils/external-readme');
 const { getAuthCredentialTemplate } = require('../utils/authentication-shape');
+const {
+  getCwdIntegrationRoot,
+  getCwdBuilderRoot,
+  getAppsMaterializationParent
+} = require('../utils/paths');
+const {
+  SYSTEM_KIND_ENTERPRISE_KNOWLEDGE,
+  getCwdEnterpriseKnowledgeRoot,
+  getEnterpriseKnowledgeRoot
+} = require('../utils/enterprise-knowledge-paths');
+const {
+  SYSTEM_KIND_DIGITAL_WORKER,
+  getCwdRoleAssistantRoot,
+  getRoleAssistantRoot
+} = require('../utils/role-assistant-paths');
+const path = require('path');
+
+/** Runtime/API fields that must not appear on external-system manifests. */
+const NON_MANIFEST_SYSTEM_KEYS = ['showOpenApiDocs'];
+
+/**
+ * True when cwd looks like a product/workspace repo (integration, builder, or EK tree).
+ * @returns {boolean}
+ */
+function cwdHasWorkspaceTree() {
+  return Boolean(
+    getCwdIntegrationRoot() ||
+    getCwdBuilderRoot() ||
+    getCwdEnterpriseKnowledgeRoot() ||
+    getCwdRoleAssistantRoot()
+  );
+}
 
 /**
  * Validates system type from downloaded application
@@ -68,6 +101,61 @@ function validateDownloadedData(application, dataSources) {
   }
 }
 
+/**
+ * Strip dataplane runtime fields that are not part of external-system.schema.json.
+ * @param {Object} application
+ * @returns {Object}
+ */
+function sanitizeDownloadedApplication(application) {
+  if (!application || typeof application !== 'object') {
+    return application;
+  }
+  const cleaned = { ...application };
+  for (const key of NON_MANIFEST_SYSTEM_KEYS) {
+    delete cleaned[key];
+  }
+  return cleaned;
+}
+
+/**
+ * Download destination:
+ * - If cwd has `integration/`, `builder/`, or `enterprise-knowledge/` → write under cwd
+ * - Else → `aifabrix-work` / materialization parent
+ * Ignores materialization orphans when cwd has a workspace tree so re-download lands in-repo.
+ * @param {string} systemKey
+ * @param {Object} application
+ * @returns {string}
+ */
+function resolveDownloadDestinationPath(systemKey, application) {
+  const kind =
+    application && typeof application.systemKind === 'string'
+      ? application.systemKind.trim()
+      : '';
+  const useCwd = cwdHasWorkspaceTree();
+  if (kind === SYSTEM_KIND_DIGITAL_WORKER) {
+    if (useCwd) {
+      const cwdRoleAssistant = getCwdRoleAssistantRoot();
+      const roleAssistantRoot = cwdRoleAssistant || path.join(path.resolve(process.cwd()), 'role-assistant');
+      return path.join(roleAssistantRoot, systemKey);
+    }
+    return path.join(getRoleAssistantRoot(), systemKey);
+  }
+  if (kind === SYSTEM_KIND_ENTERPRISE_KNOWLEDGE) {
+    if (useCwd) {
+      const cwdEk = getCwdEnterpriseKnowledgeRoot();
+      const ekRoot = cwdEk || path.join(path.resolve(process.cwd()), 'enterprise-knowledge');
+      return path.join(ekRoot, systemKey);
+    }
+    return path.join(getEnterpriseKnowledgeRoot(), systemKey);
+  }
+  if (useCwd) {
+    const cwdInt = getCwdIntegrationRoot();
+    const intRoot = cwdInt || path.join(path.resolve(process.cwd()), 'integration');
+    return path.join(intRoot, systemKey);
+  }
+  return path.join(getAppsMaterializationParent(), 'integration', systemKey);
+}
+
 function logDownloadCommandHeader(systemKey) {
   logger.log('');
   logger.log(sectionTitle('Download'));
@@ -83,13 +171,31 @@ function displayDownloadSuccess(systemKey, finalPath, datasourceCount) {
   logger.log(headerKeyValue('Location:', finalPath));
 }
 
+/**
+ * Convert YAML components to JSON after download. On validation failure, keep takeout
+ * and warn — do not discard downloaded files.
+ * @param {string} systemKey
+ * @returns {Promise<{converted: boolean, error?: string}>}
+ */
 async function runConvertToJsonIfRequested(systemKey) {
   const { runConvert } = require('../commands/convert');
   try {
     await runConvert(systemKey, { format: 'json', force: true });
     logger.log(formatSuccessLine('Converted component files to JSON'));
+    return { converted: true };
   } catch (convertErr) {
-    throw new Error(`Download succeeded but convert to JSON failed: ${convertErr.message}`);
+    const message = convertErr && convertErr.message ? convertErr.message : String(convertErr);
+    logger.log(
+      formatWarningLine(
+        `Download saved files, but convert to JSON failed: ${message}`
+      )
+    );
+    logger.log(
+      formatWarningLine(
+        'Fix schema errors in the downloaded files, then run: aifabrix convert <systemKey> --format json'
+      )
+    );
+    return { converted: false, error: message };
   }
 }
 
@@ -183,10 +289,13 @@ function generateReadme(systemKey, application, dataSources) {
 module.exports = {
   validateSystemType,
   validateDownloadedData,
+  sanitizeDownloadedApplication,
+  resolveDownloadDestinationPath,
   logDownloadCommandHeader,
   displayDownloadSuccess,
   runConvertToJsonIfRequested,
   generateVariablesYaml,
-  generateReadme
+  generateReadme,
+  NON_MANIFEST_SYSTEM_KEYS
 };
 

package/lib/external-system/download-role-assistant.js

@@ -0,0 +1,55 @@
+/**
+ * Role Assistant sidecar handling during external-system download.
+ *
+ * @fileoverview Post-download evidence/knowledge/workflow actions for digitalWorker systems
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const logger = require('../utils/logger');
+const { formatWarningLine } = require('../utils/cli-test-layout-chalk');
+const { exportRoleAssistantKnowledge, logKnowledgeExportSummary } = require('../role-assistant/knowledge-sync');
+const { exportRoleAssistantEvidence, logRoleAssistantEvidenceExportSummary } = require('../role-assistant/evidence-sync');
+
+async function maybeExportRoleAssistantEvidenceArtifacts(systemKey, manifest, dataplaneUrl, authConfig, options) {
+  try {
+    const evidenceResult = await exportRoleAssistantEvidence({
+      systemKey,
+      dataplaneUrl,
+      authConfig,
+      manifest,
+      pathOptions: options.pathOptions || {}
+    });
+    logRoleAssistantEvidenceExportSummary(systemKey, evidenceResult, { verbose: options.verbose === true });
+  } catch (evidenceErr) {
+    logger.log(formatWarningLine(`Role assistant evidence export skipped: ${evidenceErr.message}`));
+  }
+}
+
+async function maybeExportRoleAssistantKnowledgeArtifacts(systemKey, manifest, dataplaneUrl, authConfig, options) {
+  try {
+    const knowledgeResult = await exportRoleAssistantKnowledge({
+      systemKey,
+      dataplaneUrl,
+      authConfig,
+      manifest,
+      environment: options.env || options.environment || 'dev',
+      pathOptions: options.pathOptions || {}
+    });
+    logKnowledgeExportSummary(systemKey, knowledgeResult, { verbose: options.verbose === true });
+  } catch (knowledgeErr) {
+    logger.log(formatWarningLine(`Role assistant knowledge export skipped: ${knowledgeErr.message}`));
+  }
+}
+
+async function runRoleAssistantDownloadSidecars(params) {
+  const { systemKey, manifest, dataplaneUrl, authConfig, options = {} } = params;
+  await maybeExportRoleAssistantEvidenceArtifacts(systemKey, manifest, dataplaneUrl, authConfig, options);
+  await maybeExportRoleAssistantKnowledgeArtifacts(systemKey, manifest, dataplaneUrl, authConfig, options);
+}
+
+module.exports = {
+  runRoleAssistantDownloadSidecars
+};

package/lib/external-system/download.js

@@ -32,7 +32,6 @@ const yaml = require('js-yaml');
 const { getExternalSystemConfig } = require('../api/external-systems.api');
 const { getDeploymentAuth, requireBearerForDataplanePipeline } = require('../utils/token-manager');
 const { getConfig } = require('../core/config');
-const { getIntegrationPath } = require('../utils/paths');
 const { retemplateConfigurationForDownload, retemplateAllVariableDefinitions } = require('../utils/configuration-env-resolver');
 const { syncRuntimeConfigurationFromDataplane } = require('../utils/runtime-configuration-download');
 const logger = require('../utils/logger');
@@ -41,10 +40,13 @@ const generator = require('../generator');
 const {
   validateSystemType,
   validateDownloadedData,
+  sanitizeDownloadedApplication,
+  resolveDownloadDestinationPath,
   logDownloadCommandHeader,
   displayDownloadSuccess,
   runConvertToJsonIfRequested
 } = require('./download-helpers');
+const { runRoleAssistantDownloadSidecars } = require('./download-role-assistant');
 
 /**
  * Setup authentication and get dataplane URL
@@ -117,7 +119,11 @@ async function downloadFullManifest(dataplaneUrl, systemKey, authConfig, options
     throw new Error('Application configuration not found in download response');
   }
 
-  return { application, dataSources, version };
+  return {
+    application: sanitizeDownloadedApplication(application),
+    dataSources,
+    version
+  };
 }
 
 /**
@@ -304,9 +310,8 @@ async function applyRetemplateToSystemFile(systemKey, systemFilePath) {
  * @returns {Promise<string>} Final destination path
  * @throws {Error} If processing fails
  */
-async function processDownloadedSystem(systemKey, manifest, splitOptions = {}) {
+async function processDownloadedSystem(systemKey, manifest, finalPath, splitOptions = {}) {
   const { application, dataSources, version } = manifest;
-  const finalPath = getIntegrationPath(systemKey);
 
   logger.log(formatProgress('Creating integration directory…'));
   logger.log(metadata(finalPath));
@@ -331,6 +336,28 @@ async function processDownloadedSystem(systemKey, manifest, splitOptions = {}) {
   return finalPath;
 }
 
+async function maybeSyncRuntimeConfigurationForDownload(systemKey, manifest, finalPath, dataplaneUrl, authConfig) {
+  try {
+    const configSync = await syncRuntimeConfigurationFromDataplane({
+      dataplaneUrl,
+      authConfig,
+      systemKey,
+      integrationPath: finalPath,
+      dataSources: manifest.dataSources
+    });
+    if (configSync.merged > 0) {
+      const nameList = configSync.names.length ? ` (${configSync.names.join(', ')})` : '';
+      logger.log(
+        formatSuccessLine(
+          `Merged ${configSync.merged} runtime configuration variable(s) into env.template${nameList}.`
+        )
+      );
+    }
+  } catch (syncErr) {
+    logger.log(formatWarningLine(`Runtime configuration sync skipped: ${syncErr.message}`));
+  }
+}
+
 /**
  * Post-split download steps: governance packs, optional JSON convert, runtime config sync.
  * @param {Object} params
@@ -348,7 +375,8 @@ async function finalizeDownloadArtifacts({
   finalPath,
   dataplaneUrl,
   authConfig,
-  format
+  format,
+  options = {}
 }) {
   const { downloadGovernanceScenarioPacksFromManifest } = require('../lifecycle/scenario-upload');
   await downloadGovernanceScenarioPacksFromManifest(
@@ -360,25 +388,14 @@ async function finalizeDownloadArtifacts({
   if (format === 'json') {
     await runConvertToJsonIfRequested(systemKey);
   }
-  try {
-    const configSync = await syncRuntimeConfigurationFromDataplane({
-      dataplaneUrl,
-      authConfig,
-      systemKey,
-      integrationPath: finalPath,
-      dataSources: manifest.dataSources
-    });
-    if (configSync.merged > 0) {
-      const nameList = configSync.names.length ? ` (${configSync.names.join(', ')})` : '';
-      logger.log(
-        formatSuccessLine(
-          `Merged ${configSync.merged} runtime configuration variable(s) into env.template${nameList}.`
-        )
-      );
-    }
-  } catch (syncErr) {
-    logger.log(formatWarningLine(`Runtime configuration sync skipped: ${syncErr.message}`));
-  }
+  await maybeSyncRuntimeConfigurationForDownload(systemKey, manifest, finalPath, dataplaneUrl, authConfig);
+  await runRoleAssistantDownloadSidecars({
+    systemKey,
+    manifest,
+    dataplaneUrl,
+    authConfig,
+    options
+  });
   displayDownloadSuccess(systemKey, finalPath, manifest.dataSources.length);
 }
 
@@ -408,10 +425,11 @@ async function downloadExternalSystem(systemKey, options = {}) {
 
     const manifest = await downloadFullManifest(dataplaneUrl, systemKey, authConfig, options);
     validateAndLogDownloadedData(manifest.application, manifest.dataSources);
-    const finalPath = getIntegrationPath(systemKey);
+    const finalPath = resolveDownloadDestinationPath(systemKey, manifest.application);
     await processDownloadedSystem(
       systemKey,
       manifest,
+      finalPath,
       await resolveDownloadSplitOptions(finalPath, options)
     );
     await finalizeDownloadArtifacts({
@@ -420,7 +438,8 @@ async function downloadExternalSystem(systemKey, options = {}) {
       finalPath,
       dataplaneUrl,
       authConfig,
-      format
+      format,
+      options
     });
   } catch (error) {
     throw new Error(`Failed to download external system: ${error.message}`);

package/lib/external-system/test-helpers.js

@@ -78,7 +78,7 @@ function validateSystemFilesForTest(systemFiles, results) {
  * @param {Function} determineDatasourcesToTest - Function to determine datasources to test
  */
 function validateDatasourceFilesForTest(datasourceFiles, systemFiles, results, options, validateSingleDatasource, determineDatasourcesToTest) {
-  const datasourcesToTest = determineDatasourcesToTest(datasourceFiles, options.datasource);
+  const datasourcesToTest = determineDatasourcesToTest(datasourceFiles, options.datasource, options);
   const systemKey = systemFiles.length > 0 ? systemFiles[0].data.key : null;
 
   for (const datasourceFile of datasourcesToTest) {

package/lib/external-system/test.js

@@ -39,6 +39,20 @@ const { runSystemLevelTest } = require('./test-system-level');
 const { shouldUseSystemLevelIntegrationCall } = require('./integration-test-dispatch');
 const { cliOptsSkipPublish } = require('../utils/cli-sync-options');
 
+const EMPTY_DATASOURCE_ALLOWED_KINDS = new Set(['enterpriseKnowledge', 'digitalWorker']);
+
+function resolvePrimarySystemKind(systemFiles) {
+  if (!Array.isArray(systemFiles) || systemFiles.length === 0) {
+    return '';
+  }
+  const kind = systemFiles[0] && systemFiles[0].data ? systemFiles[0].data.systemKind : '';
+  return typeof kind === 'string' ? kind.trim() : '';
+}
+
+function allowEmptyDatasourcesForKind(systemKind) {
+  return EMPTY_DATASOURCE_ALLOWED_KINDS.has(String(systemKind || '').trim());
+}
+
 /**
  * Loads and parses application config file
  * @async
@@ -320,11 +334,25 @@ async function testExternalSystem(appName, options = {}) {
 
     // Load files
     const { variables: _variables, systemFiles, datasourceFiles } = await loadExternalSystemFiles(appName);
+    const systemKind = resolvePrimarySystemKind(systemFiles);
+    const allowEmptyDatasources = allowEmptyDatasourcesForKind(systemKind);
 
     const results = initializeTestResults();
 
     validateSystemFilesForTest(systemFiles, results);
-    validateDatasourceFilesForTest(datasourceFiles, systemFiles, results, options, validateSingleDatasource, determineDatasourcesToTest);
+    validateDatasourceFilesForTest(
+      datasourceFiles,
+      systemFiles,
+      results,
+      { ...options, allowEmptyDatasources, systemKindForEmptyDatasources: systemKind },
+      validateSingleDatasource,
+      determineDatasourcesToTest
+    );
+    if (allowEmptyDatasources && results.datasourceResults.length === 0) {
+      results.warnings.push(
+        `No datasources defined; integration validation is N/A for systemKind "${systemKind}".`
+      );
+    }
 
     results.systemKey = systemFiles.length > 0 ? systemFiles[0].data.key : appName;
     results.appName = appName;
@@ -383,12 +411,15 @@ async function testExternalSystem(appName, options = {}) {
  * @returns {Array} Filtered datasource files
  * @throws {Error} If no datasources found
  */
-function determineDatasourcesToTest(datasourceFiles, datasourceFilter) {
+function determineDatasourcesToTest(datasourceFiles, datasourceFilter, options = {}) {
   const datasourcesToTest = datasourceFilter
     ? datasourceFiles.filter(ds => ds.data.key === datasourceFilter || path.basename(ds.path).includes(datasourceFilter))
     : datasourceFiles;
 
   if (datasourcesToTest.length === 0) {
+    if (options.allowEmptyDatasources === true) {
+      return [];
+    }
     throw new Error('No datasources found to test');
   }
 
@@ -429,13 +460,18 @@ async function prepareIntegrationTestEnvironment(appName, options) {
   }
 
   const systemKey = systemFiles[0].data.key;
+  const systemKind = resolvePrimarySystemKind(systemFiles);
+  const allowEmptyDatasources = allowEmptyDatasourcesForKind(systemKind);
 
   // Setup authentication and dataplane URL
   const config = await getConfig();
   const { authConfig, dataplaneUrl } = await setupIntegrationTestAuth(appName, options, config);
 
   // Determine datasources to test
-  const datasourcesToTest = determineDatasourcesToTest(datasourceFiles, options.datasource);
+  const datasourcesToTest = determineDatasourcesToTest(datasourceFiles, options.datasource, {
+    allowEmptyDatasources,
+    systemKindForEmptyDatasources: systemKind
+  });
 
   // Load custom payload if provided
   const customPayload = await testHelpers.loadCustomPayload(options.payload);

package/lib/generator/builders.js

@@ -276,6 +276,9 @@ function validateBuildFields(build) {
   if (typeof build.imageRunBeat === 'string' && build.imageRunBeat.trim()) {
     buildConfig.imageRunBeat = build.imageRunBeat.trim();
   }
+  if (typeof build.deployJob === 'string' && build.deployJob.trim()) {
+    buildConfig.deployJob = build.deployJob.trim();
+  }
 
   return Object.keys(buildConfig).length > 0 ? buildConfig : null;
 }

package/lib/generator/helpers.js

@@ -156,10 +156,10 @@ function validatePortalInput(portalInput, variableName) {
  * @throws {Error} If portalInput structure is invalid
  */
 /**
- * Creates a map of portalInput configurations from variables config
+ * Creates a map of portal overlay metadata from variables config
  * @function createPortalInputMap
  * @param {Object|null} variablesConfig - Configuration from application.yaml
- * @returns {Map} Map of variable names to portalInput configurations
+ * @returns {Map} Map of variable names to { portalInput, category? }
  */
 function createPortalInputMap(variablesConfig) {
   const portalInputMap = new Map();
@@ -168,7 +168,11 @@ function createPortalInputMap(variablesConfig) {
       if (configItem.name && configItem.portalInput) {
         // Validate portalInput before adding to map
         validatePortalInput(configItem.portalInput, configItem.name);
-        portalInputMap.set(configItem.name, configItem.portalInput);
+        const overlay = { portalInput: configItem.portalInput };
+        if (typeof configItem.category === 'string' && configItem.category.trim().length > 0) {
+          overlay.category = configItem.category.trim();
+        }
+        portalInputMap.set(configItem.name, overlay);
       }
     }
   }
@@ -291,9 +295,13 @@ function createConfigItem(key, value, location, required, portalInputMap, appKey
     required
   };
 
-  // Merge portalInput if it exists in application config
+  // Merge portalInput / category if present in application config
   if (portalInputMap.has(key)) {
-    configItem.portalInput = portalInputMap.get(key);
+    const overlay = portalInputMap.get(key);
+    configItem.portalInput = overlay.portalInput;
+    if (typeof overlay.category === 'string' && overlay.category.length > 0) {
+      configItem.category = overlay.category;
+    }
   }
 
   return configItem;

package/lib/generator/split-variables.js

@@ -66,15 +66,21 @@ function extractOptionalSections(deployment) {
 }
 
 /**
- * Portal-only configuration for application.yaml (name + portalInput per entry).
+ * Portal-only configuration for application.yaml (name + optional category + portalInput).
  * @param {Array} configuration - Configuration array from deployment JSON
- * @returns {Array<{ name: string, portalInput: Object }>}
+ * @returns {Array<{ name: string, category?: string, portalInput: Object }>}
  */
 function extractPortalInputConfiguration(configuration) {
   if (!Array.isArray(configuration) || configuration.length === 0) return [];
   return configuration
     .filter(item => item && item.portalInput && typeof item.name === 'string')
-    .map(item => ({ name: item.name, portalInput: item.portalInput }));
+    .map(item => {
+      const entry = { name: item.name, portalInput: item.portalInput };
+      if (typeof item.category === 'string' && item.category.trim().length > 0) {
+        entry.category = item.category.trim();
+      }
+      return entry;
+    });
 }
 
 /**