@aifabrix/builder 2.54.1 → 2.55.0

package/lib/commands/upload-sidecar-followups.js

@@ -117,7 +117,8 @@ async function runUploadSidecarFollowups(ctx) {
     governance: null,
     evidence: null,
     knowledge: null,
-    documentStorageSettings: null
+    documentStorageSettings: null,
+    tests: null
   };
   const quietSidecar = isQuietMinimalSync(ctx.options);
   const model = buildRoleAssistantModel(ctx.payload);
@@ -131,6 +132,8 @@ async function runUploadSidecarFollowups(ctx) {
     throw err;
   }
   await runKnowledgeSidecar(ctx, model, sidecar, quietSidecar);
+  const { runTestsSidecar } = require('../role-assistant/test-pack-sync');
+  await runTestsSidecar(ctx, sidecar, quietSidecar);
   return sidecar;
 }
 

package/lib/commands/verify-operations-skip-e2e.js

@@ -0,0 +1,131 @@
+/**
+ * When verify-operations should skip live E2E (Role Assistant or static custom).
+ *
+ * @fileoverview Skip E2E for packages that have no live vendor pull
+ */
+
+'use strict';
+
+const path = require('path');
+const { loadConfigFile } = require('../utils/config-format');
+const { discoverIntegrationFiles } = require('./repair-internal');
+
+const STORAGE_ENTITY_TYPES = new Set(['recordStorage', 'documentStorage']);
+
+/**
+ * Role Assistant packages use authType none + nested builder-api CIP.
+ * Live capacity E2E expects a system baseUrl credential and is not the RA certify path.
+ * @param {string} systemKey
+ * @param {Object} options
+ * @returns {boolean}
+ */
+function shouldSkipE2eForRoleAssistant(systemKey, options) {
+  if (options.forceE2e === true) {
+    return false;
+  }
+  try {
+    const { resolveAppWorkspace } = require('../utils/app-path-resolve');
+    const { baseDir } = resolveAppWorkspace(systemKey, {});
+    return baseDir === 'role-assistant';
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * @param {string} filePath
+ * @returns {Object|null}
+ */
+function readManifestObject(filePath) {
+  try {
+    const parsed = loadConfigFile(filePath);
+    return parsed && typeof parsed === 'object' ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * @param {string} appPath
+ * @param {string} systemKey
+ * @returns {string|null}
+ */
+function readLocalSystemType(appPath, systemKey) {
+  const candidates = [
+    path.join(appPath, `${systemKey}-system.json`),
+    path.join(appPath, `${systemKey}-system.yaml`),
+    path.join(appPath, `${systemKey}-system.yml`)
+  ];
+  for (const filePath of candidates) {
+    const doc = readManifestObject(filePath);
+    if (doc && typeof doc.type === 'string' && doc.type.trim()) {
+      return doc.type.trim().toLowerCase();
+    }
+  }
+  return null;
+}
+
+/**
+ * @param {string} appPath
+ * @returns {Object[]}
+ */
+function loadStorageDatasources(appPath) {
+  const { datasourceFiles } = discoverIntegrationFiles(appPath);
+  const storage = [];
+  for (const name of datasourceFiles) {
+    const doc = readManifestObject(path.join(appPath, name));
+    if (!doc || !STORAGE_ENTITY_TYPES.has(String(doc.entityType || ''))) {
+      continue;
+    }
+    storage.push(doc);
+  }
+  return storage;
+}
+
+/**
+ * Static custom packages (syncBridge disabled) have no live vendor pull.
+ * @param {string} systemKey
+ * @param {Object} options
+ * @returns {boolean}
+ */
+function shouldSkipE2eForStaticPackage(systemKey, options) {
+  if (options.forceE2e === true) {
+    return false;
+  }
+  try {
+    const { resolveAppWorkspace } = require('../utils/app-path-resolve');
+    const { baseDir, appPath } = resolveAppWorkspace(systemKey, options);
+    if (baseDir !== 'integration' || !appPath) {
+      return false;
+    }
+    if (readLocalSystemType(appPath, systemKey) !== 'custom') {
+      return false;
+    }
+    const storage = loadStorageDatasources(appPath);
+    if (storage.length === 0) {
+      return false;
+    }
+    return storage.every((ds) => String(ds.syncBridge || 'disabled').toLowerCase() === 'disabled');
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * @param {string} systemKey
+ * @param {Object} options
+ * @returns {boolean}
+ */
+function shouldSkipOperationsE2e(systemKey, options) {
+  if (options.skipE2e === true || options.minimal === true) {
+    return true;
+  }
+  return shouldSkipE2eForRoleAssistant(systemKey, options)
+    || shouldSkipE2eForStaticPackage(systemKey, options);
+}
+
+module.exports = {
+  shouldSkipOperationsE2e,
+  shouldSkipE2eForRoleAssistant,
+  shouldSkipE2eForStaticPackage
+};

package/lib/commands/verify-operations-step-plan.js

@@ -7,6 +7,7 @@
 const { validateExternalSystemComplete } = require('../validation/validate');
 const { testExternalSystem, testExternalSystemIntegration } = require('../external-system/test');
 const { runTestE2EForExternalSystem } = require('./test-e2e-external');
+const { shouldSkipOperationsE2e } = require('./verify-operations-skip-e2e');
 
 /**
  * @param {string} systemKey
@@ -59,21 +60,13 @@ function buildAfterSyncSteps(systemKey, options) {
  * @returns {Object}
  */
 function buildE2eStep(systemKey, options) {
-  if (options.skipE2e === true || options.minimal === true) {
+  if (shouldSkipOperationsE2e(systemKey, options)) {
+    const reason = options.skipE2e === true || options.minimal === true
+      ? 'minimal'
+      : 'static-or-role-assistant';
     return {
       key: 'e2e',
-      run: async() => ({ success: true, skipped: true, reason: 'minimal' }),
-      ok: () => true
-    };
-  }
-  if (shouldSkipE2eForRoleAssistant(systemKey, options)) {
-    return {
-      key: 'e2e',
-      run: async() => ({
-        success: true,
-        skipped: true,
-        reason: 'role-assistant-auth-none'
-      }),
+      run: async() => ({ success: true, skipped: true, reason }),
       ok: () => true
     };
   }
@@ -99,26 +92,6 @@ function buildE2eStep(systemKey, options) {
   };
 }
 
-/**
- * Role Assistant packages use authType none + nested builder-api CIP.
- * Live capacity E2E expects a system baseUrl credential and is not the RA certify path.
- * @param {string} systemKey
- * @param {Object} options
- * @returns {boolean}
- */
-function shouldSkipE2eForRoleAssistant(systemKey, options) {
-  if (options.forceE2e === true) {
-    return false;
-  }
-  try {
-    const { resolveAppWorkspace } = require('../utils/app-path-resolve');
-    const { baseDir } = resolveAppWorkspace(systemKey, {});
-    return baseDir === 'role-assistant';
-  } catch {
-    return false;
-  }
-}
-
 /**
  * @param {string} systemKey
  * @param {Object} options

package/lib/commands/verify-operations-steps.js

@@ -86,7 +86,9 @@ async function runOperationsVerifySteps(systemKey, options) {
     }
   }
 
-  if (!cliOptsSkipPublish(options)) {
+  // Upload is not publish. Verify against the already-deployed dataplane unless
+  // the caller opts in with upload: true (and did not pass --no-publish).
+  if (options.upload === true && !cliOptsSkipPublish(options)) {
     const uploadCtx = await syncLocalIfRequested(systemKey, options);
     attachUploadAuthToOptions(options, uploadCtx);
   }

package/lib/datasource/capability/relate-operations.js

@@ -78,23 +78,41 @@ function _buildRelationMetadataSchema({ relationName, targetDoc, resolvedTargetF
   return out;
 }
 
+function relationNameFromCarrier(fields) {
+  const raw = String((Array.isArray(fields) && fields[0]) || '').trim();
+  if (!raw) {
+    return '';
+  }
+  if (/^[a-z][a-zA-Z0-9]*$/.test(raw)) {
+    return raw;
+  }
+  const camel = raw.replace(/[-_]([a-zA-Z0-9])/g, (_, ch) => String(ch).toUpperCase());
+  return /^[a-z][a-zA-Z0-9]*$/.test(camel) ? camel : '';
+}
+
 /**
  * FK `name` must match schema: ^[a-z][a-zA-Z0-9]*$
+ * Defaults to the published carrier (fields[0]) when --relation-name is omitted.
  *
  * @param {string} raw
+ * @param {string[]} [fields]
  * @returns {string}
  */
-function normalizeRelationName(raw) {
+function normalizeRelationName(raw, fields) {
   const s = String(raw || '').trim();
-  if (!s) {
-    throw new Error('--relation-name is required');
+  if (s) {
+    if (!/^[a-z][a-zA-Z0-9]*$/.test(s)) {
+      throw new Error(
+        `--relation-name "${s}" must match ^[a-z][a-zA-Z0-9]*$ (camelCase)`
+      );
+    }
+    return s;
   }
-  if (!/^[a-z][a-zA-Z0-9]*$/.test(s)) {
-    throw new Error(
-      `--relation-name "${s}" must match ^[a-z][a-zA-Z0-9]*$ (camelCase)`
-    );
+  const fromField = relationNameFromCarrier(fields);
+  if (fromField) {
+    return fromField;
   }
-  return s;
+  throw new Error('--relation-name is required when --field is not a camelCase carrier');
 }
 
 /**
@@ -287,9 +305,9 @@ function buildForeignKeyRow(name, fields, targetDatasource, targetFields, option
  * }}
  */
 function applyCapabilityRelate(doc, opts) {
-  const name = normalizeRelationName(opts.relationName);
-  const targetDatasource = normalizeTargetDatasource(opts.targetDatasource);
   const { fields, targetFields } = parseRelateFields(opts);
+  const name = normalizeRelationName(opts.relationName, fields);
+  const targetDatasource = normalizeTargetDatasource(opts.targetDatasource);
 
   const d = deepClone(doc);
   if (!Array.isArray(d.foreignKeys)) {

package/lib/datasource/capability/run-capability-relate.js

@@ -25,13 +25,18 @@ function loadTargetDocLocal(targetDatasourceKey) {
   return localTarget.ok ? readJsonFile(localTarget.path) : null;
 }
 
-async function maybeFetchRemoteTargetConfig({ systemKeyForAuth, targetDatasourceKey }) {
+async function maybeFetchRemoteTargetConfig({
+  systemKeyForAuth,
+  targetSystemKey,
+  targetDatasourceKey
+}) {
   const remoteFetchMeta = { attempted: false, ok: false, notAuthenticated: false };
-  if (!systemKeyForAuth) {
+  const authKey = String(targetSystemKey || systemKeyForAuth || '').trim();
+  if (!authKey) {
     return { remoteTargetConfig: null, remoteFetchMeta };
   }
   remoteFetchMeta.attempted = true;
-  const remote = await tryFetchDatasourceConfig(systemKeyForAuth, targetDatasourceKey, { silent: true });
+  const remote = await tryFetchDatasourceConfig(authKey, targetDatasourceKey, { silent: true });
   if (remote.ok) {
     return {
       remoteTargetConfig: remote.datasourceConfig,
@@ -105,6 +110,38 @@ function applyRelateAndValidateSchema(parsed, opts, semanticWarnings, remoteVali
   return { result, validation };
 }
 
+function persistOrDryRunRelate({
+  opts,
+  resolvedPath,
+  result,
+  validation,
+  semantic,
+  remoteFetchMeta
+}) {
+  if (opts.dryRun) {
+    return buildRelateResultEnvelope({
+      dryRun: true,
+      resolvedPath,
+      result,
+      backupPath: null,
+      validation,
+      semanticWarnings: semantic.warnings,
+      remoteValidation: remoteFetchMeta
+    });
+  }
+  const backupPath = writeBackup(resolvedPath, Boolean(opts.noBackup));
+  atomicWriteJson(resolvedPath, result.doc);
+  return buildRelateResultEnvelope({
+    dryRun: false,
+    resolvedPath,
+    result,
+    backupPath,
+    validation,
+    semanticWarnings: semantic.warnings,
+    remoteValidation: remoteFetchMeta
+  });
+}
+
 /**
  * @typedef {object} RunCapabilityRelateOpts
  * @property {string} fileOrKey
@@ -137,8 +174,10 @@ async function runCapabilityRelate(opts) {
 
   const targetDocLocal = loadTargetDocLocal(opts.targetDatasource);
   const systemKeyForAuth = resolveSystemKeyForAuth(parsed);
+  const targetSystemKey = resolveSystemKeyForAuth(targetDocLocal) || systemKeyForAuth;
   const { remoteTargetConfig, remoteFetchMeta } = await maybeFetchRemoteTargetConfig({
     systemKeyForAuth,
+    targetSystemKey,
     targetDatasourceKey: opts.targetDatasource
   });
   const semantic = runSemanticValidation({
@@ -159,33 +198,17 @@ async function runCapabilityRelate(opts) {
     semantic.warnings,
     remoteFetchMeta
   );
-
-  if (opts.dryRun) {
-    return buildRelateResultEnvelope({
-      dryRun: true,
-      resolvedPath,
-      result,
-      backupPath: null,
-      validation,
-      semanticWarnings: semantic.warnings,
-      remoteValidation: remoteFetchMeta
-    });
-  }
-
-  const backupPath = writeBackup(resolvedPath, Boolean(opts.noBackup));
-  atomicWriteJson(resolvedPath, result.doc);
-
-  return buildRelateResultEnvelope({
-    dryRun: false,
+  return persistOrDryRunRelate({
+    opts,
     resolvedPath,
     result,
-    backupPath,
     validation,
-    semanticWarnings: semantic.warnings,
-    remoteValidation: remoteFetchMeta
+    semantic,
+    remoteFetchMeta
   });
 }
 
 module.exports = {
-  runCapabilityRelate
+  runCapabilityRelate,
+  resolveSystemKeyForAuth
 };

package/lib/datasource/datasource-identifier.js

@@ -56,12 +56,20 @@ function pickIntegrationAppForDatasourceIdentifier(identifier, appNames) {
   if (!id) {
     return null;
   }
-  const matches = appNames.filter(app => id === app || id.startsWith(`${app}-`));
-  if (matches.length === 0) {
-    return null;
+  const prefixMatches = (appNames || []).filter((app) => id === app || id.startsWith(`${app}-`));
+  if (prefixMatches.length > 0) {
+    prefixMatches.sort((a, b) => b.length - a.length);
+    return prefixMatches[0];
+  }
+  const keyMatches = (appNames || []).filter((app) => appHasDatasourceIdentifier(app, id));
+  if (keyMatches.length === 1) {
+    return keyMatches[0];
+  }
+  if (keyMatches.length > 1) {
+    keyMatches.sort((a, b) => b.length - a.length);
+    return keyMatches[0];
   }
-  matches.sort((a, b) => b.length - a.length);
-  return matches[0];
+  return null;
 }
 
 /**

package/lib/deployment/deployer-poll-run.js

@@ -18,52 +18,133 @@ const {
 const { resolvePollIntervalFromController } = require('./poll-interval');
 const { createDeployPollHandlers } = require('./deploy-poll-ui');
 
+/** Same progress for this many polls + dataplane published → treat deploy as complete. */
+const STUCK_PROGRESS_POLLS = 10;
+
+function recordStuckProgress(state, snapshot) {
+  const progress = Number(snapshot && snapshot.progress);
+  const value = Number.isFinite(progress) ? progress : null;
+  if (value === state.lastProgress) {
+    state.stuckCount += 1;
+  } else {
+    state.lastProgress = value;
+    state.stuckCount = 1;
+  }
+  return state;
+}
+
+async function completeIfDataplanePublished(lastSeen, options) {
+  if (!lastSeen || typeof options.isDataplanePublished !== 'function') {
+    return null;
+  }
+  try {
+    if (await options.isDataplanePublished()) {
+      return {
+        ...lastSeen,
+        status: 'completed',
+        progress: 100,
+        completedViaDataplane: true
+      };
+    }
+  } catch {
+    return null;
+  }
+  return null;
+}
+
 /**
- * Polls deployment status from controller.
- *
- * @async
- * @param {string} deploymentId
- * @param {string} controllerUrl
- * @param {string} envKey
- * @param {Object} authConfig
- * @param {Object} [options]
- * @returns {Promise<Object>}
+ * Poll until the controller job is terminal, or dataplane is already published.
+ * @param {Object} ctx
+ * @returns {Promise<Object|null>}
  */
+async function pollUntilTerminal(ctx) {
+  const {
+    controllerUrl,
+    validatedEnvKey,
+    deploymentId,
+    pipelineAuthConfig,
+    interval,
+    maxAttempts,
+    onPollWait,
+    wrappedProgress,
+    stuckState,
+    options,
+    getLastSeen
+  } = ctx;
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    if (onPollWait) {
+      onPollWait(attempt, maxAttempts);
+    }
+    const response = await getPipelineDeployment(
+      controllerUrl,
+      validatedEnvKey,
+      deploymentId,
+      pipelineAuthConfig
+    );
+    const deploymentData = await processDeploymentStatusResponse(
+      response,
+      attempt,
+      maxAttempts,
+      interval,
+      deploymentId,
+      wrappedProgress
+    );
+    if (deploymentData) {
+      return deploymentData;
+    }
+    if (stuckState.stuckCount >= STUCK_PROGRESS_POLLS) {
+      const viaDataplane = await completeIfDataplanePublished(getLastSeen(), options);
+      if (viaDataplane) {
+        return viaDataplane;
+      }
+    }
+  }
+  return null;
+}
+
 async function pollDeploymentStatus(deploymentId, controllerUrl, envKey, authConfig, options = {}) {
   const interval = options.interval || 5000;
   const maxAttempts = options.maxAttempts || 300;
-
   const validatedEnvKey = validateEnvironmentKey(envKey);
   const pipelineAuthConfig = convertToPipelineAuthConfig(authConfig);
-
   const onPollProgress = typeof options.onPollProgress === 'function' ? options.onPollProgress : null;
   const onPollWait = typeof options.onPollWait === 'function' ? options.onPollWait : null;
-
-  for (let attempt = 0; attempt < maxAttempts; attempt++) {
-    try {
-      if (onPollWait) {
-        onPollWait(attempt, maxAttempts);
-      }
-      const response = await getPipelineDeployment(controllerUrl, validatedEnvKey, deploymentId, pipelineAuthConfig);
-      const deploymentData = await processDeploymentStatusResponse(
-        response,
-        attempt,
-        maxAttempts,
-        interval,
-        deploymentId,
-        onPollProgress
-      );
-      if (deploymentData) {
-        return deploymentData;
-      }
-    } catch (error) {
-      if (error.message && error.message.includes('not found')) {
-        throw error;
-      }
+  const stuckState = { lastProgress: undefined, stuckCount: 0 };
+  let lastSeen = null;
+  const wrappedProgress = (data, attempt, max) => {
+    lastSeen = data;
+    recordStuckProgress(stuckState, data);
+    if (onPollProgress) {
+      onPollProgress(data, attempt, max);
+    }
+  };
+  try {
+    const terminal = await pollUntilTerminal({
+      controllerUrl,
+      validatedEnvKey,
+      deploymentId,
+      pipelineAuthConfig,
+      interval,
+      maxAttempts,
+      onPollWait,
+      wrappedProgress,
+      stuckState,
+      options,
+      getLastSeen: () => lastSeen
+    });
+    if (terminal) {
+      return terminal;
+    }
+  } catch (error) {
+    if (error.message && error.message.includes('not found')) {
       throw error;
     }
+    throw error;
+  }
+  const timedOutComplete = await completeIfDataplanePublished(lastSeen, options);
+  if (timedOutComplete) {
+    return timedOutComplete;
   }
-
   throw new Error('Deployment timeout: Maximum polling attempts reached');
 }
 
@@ -94,7 +175,8 @@ async function pollDeployment(result, url, validatedEnvKey, authConfig, options)
         interval: resolvedInterval,
         maxAttempts,
         onPollProgress: pollUi.onPollProgress,
-        onPollWait: pollUi.onPollWait
+        onPollWait: pollUi.onPollWait,
+        isDataplanePublished: options.isDataplanePublished
       }
     );
 

package/lib/external-system/deploy-dataplane-published.js

@@ -0,0 +1,70 @@
+/**
+ * Treat a stuck controller deploy poll as complete when dataplane is already published.
+ *
+ * @fileoverview Dataplane published check for external deploy poll
+ */
+
+'use strict';
+
+const { getExternalSystem } = require('../api/external-systems.api');
+const { unwrapApiData } = require('../utils/external-system-readiness-core');
+const { resolveDataplaneUrl } = require('../utils/dataplane-resolver');
+
+/**
+ * @param {Object|null|undefined} system
+ * @returns {boolean}
+ */
+function isDataplaneSystemPublished(system) {
+  const status = String((system && system.status) || '').toLowerCase();
+  return status === 'published' || status === 'deployed';
+}
+
+/**
+ * @param {string} controllerUrl
+ * @param {string} environment
+ * @param {Object} authConfig
+ * @param {string} systemKey
+ * @param {string} [dataplaneUrl]
+ * @returns {Promise<boolean>}
+ */
+async function checkDataplaneSystemPublished(
+  controllerUrl,
+  environment,
+  authConfig,
+  systemKey,
+  dataplaneUrl
+) {
+  let url = dataplaneUrl;
+  if (!url) {
+    url = await resolveDataplaneUrl(controllerUrl, environment, authConfig, { silent: true });
+  }
+  const getRes = await getExternalSystem(url, systemKey, authConfig);
+  return isDataplaneSystemPublished(unwrapApiData(getRes));
+}
+
+/**
+ * @param {Object} params
+ * @returns {() => Promise<boolean>}
+ */
+function createDataplanePublishedCheck(params) {
+  const { controllerUrl, environment, authConfig, systemKey, dataplaneUrl } = params;
+  return async function isDataplanePublished() {
+    try {
+      return await checkDataplaneSystemPublished(
+        controllerUrl,
+        environment,
+        authConfig,
+        systemKey,
+        dataplaneUrl
+      );
+    } catch {
+      return false;
+    }
+  };
+}
+
+module.exports = {
+  isDataplaneSystemPublished,
+  checkDataplaneSystemPublished,
+  createDataplanePublishedCheck
+};