@aifabrix/builder 2.53.1 → 2.55.0

package/lib/commands/verify-operations-steps.js

@@ -86,7 +86,9 @@ async function runOperationsVerifySteps(systemKey, options) {
     }
   }
 
-  if (!cliOptsSkipPublish(options)) {
+  // Upload is not publish. Verify against the already-deployed dataplane unless
+  // the caller opts in with upload: true (and did not pass --no-publish).
+  if (options.upload === true && !cliOptsSkipPublish(options)) {
     const uploadCtx = await syncLocalIfRequested(systemKey, options);
     attachUploadAuthToOptions(options, uploadCtx);
   }

package/lib/core/secrets-docker-env.js

@@ -10,7 +10,7 @@ const fs = require('fs');
 const yaml = require('js-yaml');
 const config = require('./config');
 const { getEnvHosts } = require('../utils/env-endpoints');
-const { getContainerPortFromPath } = require('../utils/port-resolver');
+const { getContainerPortFromPath, isExternalAppPath } = require('../utils/port-resolver');
 
 /**
  * Gets base docker environment config
@@ -69,6 +69,9 @@ function getContainerPortFromDockerEnv(dockerEnv) {
  * @returns {Promise<string>} Updated content with PORT set
  */
 async function updatePortForDocker(resolved, variablesPath) {
+  if (isExternalAppPath(variablesPath)) {
+    return resolved;
+  }
   let dockerEnv = await getBaseDockerEnv();
   dockerEnv = applyDockerEnvOverride(dockerEnv);
 

package/lib/core/secrets-env-content.js

@@ -29,7 +29,13 @@ const {
   applyDockerEnvOverride,
   getContainerPortFromDockerEnv
 } = require('./secrets-docker-env');
-const { getContainerPortFromPath, loadVariablesFromPath } = require('../utils/port-resolver');
+const {
+  getContainerPortFromPath,
+  loadVariablesFromPath,
+  isExternalAppDoc,
+  isExternalAppPath,
+  stripListenPortEnvLines
+} = require('../utils/port-resolver');
 const secretsEnsure = require('./secrets-ensure');
 const { resolveSecretsPath, getActualSecretsPath } = require('../utils/secrets-path');
 const pathsUtil = require('../utils/paths');
@@ -163,9 +169,14 @@ async function applyDockerTransformations(resolved, variablesPath) {
   envVars.REDIS_PORT = String(redisPort);
   envVars.DB_HOST = dbHost;
   envVars.DB_PORT = String(dbPort);
-  envVars.PORT = String(containerPort);
+  if (!isExternalAppDoc(appDoc)) {
+    envVars.PORT = String(containerPort);
+  }
   resolved = interpolateEnvVars(resolved, envVars);
   resolved = rewriteDockerManifestPublicPortEnvLine(resolved, envVars, appDoc);
+  if (isExternalAppDoc(appDoc)) {
+    return resolved;
+  }
   return updatePortForDocker(resolved, variablesPath);
 }
 
@@ -389,10 +400,13 @@ function resolveEnvContentToWrite(resolved, pathToPreserve, options = {}) {
 async function writeResolvedEnv({ appName, envPath, resolved, variablesPath, secretsPath, opts }) {
   const preservePath = opts.preserveFromPath !== undefined && opts.preserveFromPath !== null ? opts.preserveFromPath : null;
   const pathToPreserve = preservePath !== null ? preservePath : envPath;
-  const toWrite = resolveEnvContentToWrite(resolved, pathToPreserve, {
+  let toWrite = resolveEnvContentToWrite(resolved, pathToPreserve, {
     freshEnv: opts.freshEnv === true,
     pruneOrphanKv: opts.envOnly === true && opts.freshEnv !== true
   });
+  if (isExternalAppPath(variablesPath) && !/^PORT\s*=/m.test(resolved || '')) {
+    toWrite = stripListenPortEnvLines(toWrite);
+  }
   fs.writeFileSync(envPath, toWrite, { mode: 0o600 });
 
   if (!opts.skipOutputPath) {

package/lib/datasource/capability/capability-manifest.js

@@ -27,11 +27,8 @@ const DEFAULT_RISK_BY_KEY = {
 };
 
 const RESOURCE_TYPE_MAP = {
-  conversation: 'message',
   user: 'employee',
   person: 'employee',
-  record: 'document',
-  folder: 'document',
   country: 'organization',
   report: 'document',
   calendar: 'event',

package/lib/datasource/capability/copy-test-payload.js

@@ -1,5 +1,8 @@
 /**
  * Clone testPayload.scenarios rows when copying a capability.
+ * Deep-clones whole scenario objects and rewrites operation only. Nested row gold
+ * (expectedRows and related keys) is copied as-is; it may be wrong for --as —
+ * the author must edit after copy. Does not invent gold.
  *
  * @fileoverview Used by applyCapabilityCopy when includeTestPayload is true
  * @author AI Fabrix Team

package/lib/datasource/capability/relate-operations.js

@@ -78,23 +78,41 @@ function _buildRelationMetadataSchema({ relationName, targetDoc, resolvedTargetF
   return out;
 }
 
+function relationNameFromCarrier(fields) {
+  const raw = String((Array.isArray(fields) && fields[0]) || '').trim();
+  if (!raw) {
+    return '';
+  }
+  if (/^[a-z][a-zA-Z0-9]*$/.test(raw)) {
+    return raw;
+  }
+  const camel = raw.replace(/[-_]([a-zA-Z0-9])/g, (_, ch) => String(ch).toUpperCase());
+  return /^[a-z][a-zA-Z0-9]*$/.test(camel) ? camel : '';
+}
+
 /**
  * FK `name` must match schema: ^[a-z][a-zA-Z0-9]*$
+ * Defaults to the published carrier (fields[0]) when --relation-name is omitted.
  *
  * @param {string} raw
+ * @param {string[]} [fields]
  * @returns {string}
  */
-function normalizeRelationName(raw) {
+function normalizeRelationName(raw, fields) {
   const s = String(raw || '').trim();
-  if (!s) {
-    throw new Error('--relation-name is required');
+  if (s) {
+    if (!/^[a-z][a-zA-Z0-9]*$/.test(s)) {
+      throw new Error(
+        `--relation-name "${s}" must match ^[a-z][a-zA-Z0-9]*$ (camelCase)`
+      );
+    }
+    return s;
   }
-  if (!/^[a-z][a-zA-Z0-9]*$/.test(s)) {
-    throw new Error(
-      `--relation-name "${s}" must match ^[a-z][a-zA-Z0-9]*$ (camelCase)`
-    );
+  const fromField = relationNameFromCarrier(fields);
+  if (fromField) {
+    return fromField;
   }
-  return s;
+  throw new Error('--relation-name is required when --field is not a camelCase carrier');
 }
 
 /**
@@ -287,9 +305,9 @@ function buildForeignKeyRow(name, fields, targetDatasource, targetFields, option
  * }}
  */
 function applyCapabilityRelate(doc, opts) {
-  const name = normalizeRelationName(opts.relationName);
-  const targetDatasource = normalizeTargetDatasource(opts.targetDatasource);
   const { fields, targetFields } = parseRelateFields(opts);
+  const name = normalizeRelationName(opts.relationName, fields);
+  const targetDatasource = normalizeTargetDatasource(opts.targetDatasource);
 
   const d = deepClone(doc);
   if (!Array.isArray(d.foreignKeys)) {

package/lib/datasource/capability/run-capability-relate.js

@@ -25,13 +25,18 @@ function loadTargetDocLocal(targetDatasourceKey) {
   return localTarget.ok ? readJsonFile(localTarget.path) : null;
 }
 
-async function maybeFetchRemoteTargetConfig({ systemKeyForAuth, targetDatasourceKey }) {
+async function maybeFetchRemoteTargetConfig({
+  systemKeyForAuth,
+  targetSystemKey,
+  targetDatasourceKey
+}) {
   const remoteFetchMeta = { attempted: false, ok: false, notAuthenticated: false };
-  if (!systemKeyForAuth) {
+  const authKey = String(targetSystemKey || systemKeyForAuth || '').trim();
+  if (!authKey) {
     return { remoteTargetConfig: null, remoteFetchMeta };
   }
   remoteFetchMeta.attempted = true;
-  const remote = await tryFetchDatasourceConfig(systemKeyForAuth, targetDatasourceKey, { silent: true });
+  const remote = await tryFetchDatasourceConfig(authKey, targetDatasourceKey, { silent: true });
   if (remote.ok) {
     return {
       remoteTargetConfig: remote.datasourceConfig,
@@ -105,6 +110,38 @@ function applyRelateAndValidateSchema(parsed, opts, semanticWarnings, remoteVali
   return { result, validation };
 }
 
+function persistOrDryRunRelate({
+  opts,
+  resolvedPath,
+  result,
+  validation,
+  semantic,
+  remoteFetchMeta
+}) {
+  if (opts.dryRun) {
+    return buildRelateResultEnvelope({
+      dryRun: true,
+      resolvedPath,
+      result,
+      backupPath: null,
+      validation,
+      semanticWarnings: semantic.warnings,
+      remoteValidation: remoteFetchMeta
+    });
+  }
+  const backupPath = writeBackup(resolvedPath, Boolean(opts.noBackup));
+  atomicWriteJson(resolvedPath, result.doc);
+  return buildRelateResultEnvelope({
+    dryRun: false,
+    resolvedPath,
+    result,
+    backupPath,
+    validation,
+    semanticWarnings: semantic.warnings,
+    remoteValidation: remoteFetchMeta
+  });
+}
+
 /**
  * @typedef {object} RunCapabilityRelateOpts
  * @property {string} fileOrKey
@@ -137,8 +174,10 @@ async function runCapabilityRelate(opts) {
 
   const targetDocLocal = loadTargetDocLocal(opts.targetDatasource);
   const systemKeyForAuth = resolveSystemKeyForAuth(parsed);
+  const targetSystemKey = resolveSystemKeyForAuth(targetDocLocal) || systemKeyForAuth;
   const { remoteTargetConfig, remoteFetchMeta } = await maybeFetchRemoteTargetConfig({
     systemKeyForAuth,
+    targetSystemKey,
     targetDatasourceKey: opts.targetDatasource
   });
   const semantic = runSemanticValidation({
@@ -159,33 +198,17 @@ async function runCapabilityRelate(opts) {
     semantic.warnings,
     remoteFetchMeta
   );
-
-  if (opts.dryRun) {
-    return buildRelateResultEnvelope({
-      dryRun: true,
-      resolvedPath,
-      result,
-      backupPath: null,
-      validation,
-      semanticWarnings: semantic.warnings,
-      remoteValidation: remoteFetchMeta
-    });
-  }
-
-  const backupPath = writeBackup(resolvedPath, Boolean(opts.noBackup));
-  atomicWriteJson(resolvedPath, result.doc);
-
-  return buildRelateResultEnvelope({
-    dryRun: false,
+  return persistOrDryRunRelate({
+    opts,
     resolvedPath,
     result,
-    backupPath,
     validation,
-    semanticWarnings: semantic.warnings,
-    remoteValidation: remoteFetchMeta
+    semantic,
+    remoteFetchMeta
   });
 }
 
 module.exports = {
-  runCapabilityRelate
+  runCapabilityRelate,
+  resolveSystemKeyForAuth
 };

package/lib/datasource/datasource-identifier.js

@@ -56,12 +56,20 @@ function pickIntegrationAppForDatasourceIdentifier(identifier, appNames) {
   if (!id) {
     return null;
   }
-  const matches = appNames.filter(app => id === app || id.startsWith(`${app}-`));
-  if (matches.length === 0) {
-    return null;
+  const prefixMatches = (appNames || []).filter((app) => id === app || id.startsWith(`${app}-`));
+  if (prefixMatches.length > 0) {
+    prefixMatches.sort((a, b) => b.length - a.length);
+    return prefixMatches[0];
+  }
+  const keyMatches = (appNames || []).filter((app) => appHasDatasourceIdentifier(app, id));
+  if (keyMatches.length === 1) {
+    return keyMatches[0];
+  }
+  if (keyMatches.length > 1) {
+    keyMatches.sort((a, b) => b.length - a.length);
+    return keyMatches[0];
   }
-  matches.sort((a, b) => b.length - a.length);
-  return matches[0];
+  return null;
 }
 
 /**

package/lib/deployment/deployer-poll-run.js

@@ -18,52 +18,133 @@ const {
 const { resolvePollIntervalFromController } = require('./poll-interval');
 const { createDeployPollHandlers } = require('./deploy-poll-ui');
 
+/** Same progress for this many polls + dataplane published → treat deploy as complete. */
+const STUCK_PROGRESS_POLLS = 10;
+
+function recordStuckProgress(state, snapshot) {
+  const progress = Number(snapshot && snapshot.progress);
+  const value = Number.isFinite(progress) ? progress : null;
+  if (value === state.lastProgress) {
+    state.stuckCount += 1;
+  } else {
+    state.lastProgress = value;
+    state.stuckCount = 1;
+  }
+  return state;
+}
+
+async function completeIfDataplanePublished(lastSeen, options) {
+  if (!lastSeen || typeof options.isDataplanePublished !== 'function') {
+    return null;
+  }
+  try {
+    if (await options.isDataplanePublished()) {
+      return {
+        ...lastSeen,
+        status: 'completed',
+        progress: 100,
+        completedViaDataplane: true
+      };
+    }
+  } catch {
+    return null;
+  }
+  return null;
+}
+
 /**
- * Polls deployment status from controller.
- *
- * @async
- * @param {string} deploymentId
- * @param {string} controllerUrl
- * @param {string} envKey
- * @param {Object} authConfig
- * @param {Object} [options]
- * @returns {Promise<Object>}
+ * Poll until the controller job is terminal, or dataplane is already published.
+ * @param {Object} ctx
+ * @returns {Promise<Object|null>}
  */
+async function pollUntilTerminal(ctx) {
+  const {
+    controllerUrl,
+    validatedEnvKey,
+    deploymentId,
+    pipelineAuthConfig,
+    interval,
+    maxAttempts,
+    onPollWait,
+    wrappedProgress,
+    stuckState,
+    options,
+    getLastSeen
+  } = ctx;
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    if (onPollWait) {
+      onPollWait(attempt, maxAttempts);
+    }
+    const response = await getPipelineDeployment(
+      controllerUrl,
+      validatedEnvKey,
+      deploymentId,
+      pipelineAuthConfig
+    );
+    const deploymentData = await processDeploymentStatusResponse(
+      response,
+      attempt,
+      maxAttempts,
+      interval,
+      deploymentId,
+      wrappedProgress
+    );
+    if (deploymentData) {
+      return deploymentData;
+    }
+    if (stuckState.stuckCount >= STUCK_PROGRESS_POLLS) {
+      const viaDataplane = await completeIfDataplanePublished(getLastSeen(), options);
+      if (viaDataplane) {
+        return viaDataplane;
+      }
+    }
+  }
+  return null;
+}
+
 async function pollDeploymentStatus(deploymentId, controllerUrl, envKey, authConfig, options = {}) {
   const interval = options.interval || 5000;
   const maxAttempts = options.maxAttempts || 300;
-
   const validatedEnvKey = validateEnvironmentKey(envKey);
   const pipelineAuthConfig = convertToPipelineAuthConfig(authConfig);
-
   const onPollProgress = typeof options.onPollProgress === 'function' ? options.onPollProgress : null;
   const onPollWait = typeof options.onPollWait === 'function' ? options.onPollWait : null;
-
-  for (let attempt = 0; attempt < maxAttempts; attempt++) {
-    try {
-      if (onPollWait) {
-        onPollWait(attempt, maxAttempts);
-      }
-      const response = await getPipelineDeployment(controllerUrl, validatedEnvKey, deploymentId, pipelineAuthConfig);
-      const deploymentData = await processDeploymentStatusResponse(
-        response,
-        attempt,
-        maxAttempts,
-        interval,
-        deploymentId,
-        onPollProgress
-      );
-      if (deploymentData) {
-        return deploymentData;
-      }
-    } catch (error) {
-      if (error.message && error.message.includes('not found')) {
-        throw error;
-      }
+  const stuckState = { lastProgress: undefined, stuckCount: 0 };
+  let lastSeen = null;
+  const wrappedProgress = (data, attempt, max) => {
+    lastSeen = data;
+    recordStuckProgress(stuckState, data);
+    if (onPollProgress) {
+      onPollProgress(data, attempt, max);
+    }
+  };
+  try {
+    const terminal = await pollUntilTerminal({
+      controllerUrl,
+      validatedEnvKey,
+      deploymentId,
+      pipelineAuthConfig,
+      interval,
+      maxAttempts,
+      onPollWait,
+      wrappedProgress,
+      stuckState,
+      options,
+      getLastSeen: () => lastSeen
+    });
+    if (terminal) {
+      return terminal;
+    }
+  } catch (error) {
+    if (error.message && error.message.includes('not found')) {
       throw error;
     }
+    throw error;
+  }
+  const timedOutComplete = await completeIfDataplanePublished(lastSeen, options);
+  if (timedOutComplete) {
+    return timedOutComplete;
   }
-
   throw new Error('Deployment timeout: Maximum polling attempts reached');
 }
 
@@ -94,7 +175,8 @@ async function pollDeployment(result, url, validatedEnvKey, authConfig, options)
         interval: resolvedInterval,
         maxAttempts,
         onPollProgress: pollUi.onPollProgress,
-        onPollWait: pollUi.onPollWait
+        onPollWait: pollUi.onPollWait,
+        isDataplanePublished: options.isDataplanePublished
       }
     );
 

package/lib/external-system/deploy-dataplane-published.js

@@ -0,0 +1,70 @@
+/**
+ * Treat a stuck controller deploy poll as complete when dataplane is already published.
+ *
+ * @fileoverview Dataplane published check for external deploy poll
+ */
+
+'use strict';
+
+const { getExternalSystem } = require('../api/external-systems.api');
+const { unwrapApiData } = require('../utils/external-system-readiness-core');
+const { resolveDataplaneUrl } = require('../utils/dataplane-resolver');
+
+/**
+ * @param {Object|null|undefined} system
+ * @returns {boolean}
+ */
+function isDataplaneSystemPublished(system) {
+  const status = String((system && system.status) || '').toLowerCase();
+  return status === 'published' || status === 'deployed';
+}
+
+/**
+ * @param {string} controllerUrl
+ * @param {string} environment
+ * @param {Object} authConfig
+ * @param {string} systemKey
+ * @param {string} [dataplaneUrl]
+ * @returns {Promise<boolean>}
+ */
+async function checkDataplaneSystemPublished(
+  controllerUrl,
+  environment,
+  authConfig,
+  systemKey,
+  dataplaneUrl
+) {
+  let url = dataplaneUrl;
+  if (!url) {
+    url = await resolveDataplaneUrl(controllerUrl, environment, authConfig, { silent: true });
+  }
+  const getRes = await getExternalSystem(url, systemKey, authConfig);
+  return isDataplaneSystemPublished(unwrapApiData(getRes));
+}
+
+/**
+ * @param {Object} params
+ * @returns {() => Promise<boolean>}
+ */
+function createDataplanePublishedCheck(params) {
+  const { controllerUrl, environment, authConfig, systemKey, dataplaneUrl } = params;
+  return async function isDataplanePublished() {
+    try {
+      return await checkDataplaneSystemPublished(
+        controllerUrl,
+        environment,
+        authConfig,
+        systemKey,
+        dataplaneUrl
+      );
+    } catch {
+      return false;
+    }
+  };
+}
+
+module.exports = {
+  isDataplaneSystemPublished,
+  checkDataplaneSystemPublished,
+  createDataplanePublishedCheck
+};

package/lib/external-system/deploy-helpers.js

@@ -170,6 +170,30 @@ async function uploadDigitalWorkerIfNeeded(appName, options, _compact, manifest)
   });
 }
 
+async function deployRoleAssistantShellIfNeeded(appName, options, baseDir) {
+  const { isRoleAssistantWorkspace } = require('../utils/platform-shell-package');
+  if (!isRoleAssistantWorkspace(baseDir)) {
+    return null;
+  }
+  const { formatSuccessParagraph } = require('../utils/cli-test-layout-chalk');
+  const logger = require('../utils/logger');
+  if (options.json !== true) {
+    logger.log(
+      formatSuccessParagraph(
+        'Role Assistant deployment uses settings and Evidence upload (no Connected System publish).'
+      )
+    );
+  }
+  const { uploadExternalSystem } = require('../commands/upload');
+  return uploadExternalSystem(appName, {
+    ...options,
+    uploadStatus: 'published',
+    silentResolve: false,
+    suppressPipelineSpinner: false,
+    syncMode: false
+  });
+}
+
 module.exports = {
   loadVariablesYaml,
   validateSingleSystemFile,
@@ -177,6 +201,7 @@ module.exports = {
   validateSingleDatasourceFile,
   validateDatasourceFiles,
   extractSystemKey,
-  uploadDigitalWorkerIfNeeded
+  uploadDigitalWorkerIfNeeded,
+  deployRoleAssistantShellIfNeeded
 };
 

package/lib/external-system/deploy.js

@@ -30,7 +30,8 @@ const { displayValidationResults } = require('../validation/validate-display');
 const { syncDeployJsonFromSources } = require('./sync-deploy-manifest');
 const { runDeployGateOrchestrator } = require('../commands/deploy-gate-orchestrator');
 const { runPostPublishDataSync } = require('../commands/run-post-publish-data-sync');
-const { uploadDigitalWorkerIfNeeded } = require('./deploy-helpers');
+const { uploadDigitalWorkerIfNeeded, deployRoleAssistantShellIfNeeded } = require('./deploy-helpers');
+const { createDataplanePublishedCheck } = require('./deploy-dataplane-published');
 
 /**
  * Validates manifest on dataplane (dry-run) before controller deploy.
@@ -194,9 +195,10 @@ const EXTERNAL_DEPLOY_POLL_INTERVAL_MS = 2000;
 /** 300 × 2s ≈ 10 minutes — must cover controller publish-job max wait with gates. */
 const EXTERNAL_DEPLOY_POLL_MAX_ATTEMPTS = 300;
 
-function buildExternalDeployPollOpts(options) {
+function buildExternalDeployPollOpts(options, extra = {}) {
   return {
     ...options,
+    ...extra,
     poll: true,
     pollInterval:
       options.pollInterval !== undefined ? options.pollInterval : EXTERNAL_DEPLOY_POLL_INTERVAL_MS,
@@ -300,7 +302,15 @@ async function executeDeployAndDisplay(manifest, controllerUrl, environment, aut
     controllerUrl,
     environment,
     authConfig,
-    buildExternalDeployPollOpts(options)
+    buildExternalDeployPollOpts(options, {
+      isDataplanePublished: createDataplanePublishedCheck({
+        controllerUrl,
+        environment,
+        authConfig,
+        systemKey: manifest.key,
+        dataplaneUrl: options.knownDataplaneUrl
+      })
+    })
   );
 
   const deploymentOutcome = parseControllerDeploymentOutcome(result);
@@ -412,17 +422,13 @@ async function validateDataplaneManifestForDeploy(manifest, authConfig, dataplan
   }
 }
 
-async function runExternalSystemDeployFlow(appName, options) {
-  const compact = options.verbose !== true;
-  const { appPath } = await detectAppType(appName);
-  logOfflinePathWhenType(appPath);
+async function runConnectedSystemDeployAfterDetect(appName, options, compact) {
   if (!compact) {
     logger.log(chalk.blue(`\n🚀 Deploying external system: ${appName}`));
   }
   await validateExternalSystemForDeploy(appName, options, compact);
   const gateOutcomes = await runDeployGateOrchestrator(appName, {
     ...options,
-    // Compact deploy still shows step progress; only JSON mode stays fully quiet.
     quietDeployUi: options.json === true
   });
   await syncDeployJsonFromSources(appName);
@@ -453,6 +459,14 @@ async function runExternalSystemDeployFlow(appName, options) {
   });
 }
 
+async function runExternalSystemDeployFlow(appName, options) {
+  const compact = options.verbose !== true;
+  const { appPath, baseDir } = await detectAppType(appName);
+  logOfflinePathWhenType(appPath);
+  const shellResult = await deployRoleAssistantShellIfNeeded(appName, options, baseDir);
+  return shellResult || runConnectedSystemDeployAfterDetect(appName, options, compact);
+}
+
 /**
  * Deploys external system via controller pipeline (same as regular apps)
  *