@aifabrix/builder 2.45.0 → 2.46.0

package/lib/commands/upload-publish-context.js

@@ -0,0 +1,75 @@
+/**
+ * @fileoverview Upload publish preflight: auth resolve, CLI gate, target logging.
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const { sectionTitle, headerKeyValue } = require('../utils/cli-test-layout-chalk');
+const { resolveControllerUrl } = require('../utils/controller-url');
+const { requireBearerForDataplanePipeline } = require('../utils/token-manager');
+const { logDataplanePipelineWarning } = require('../utils/dataplane-pipeline-warning');
+const { assertDataplaneCliVersionCompatible } = require('../utils/dataplane-cli-version-gate');
+
+const SEP = chalk.gray('────────────────────────────────────────');
+
+function logForcePublishWarning() {
+  logger.log(
+    chalk.yellow(
+      'Warning: --force bypasses active-datasource schema replacement guards on publish. '
+        + 'Existing records and ABAC may be inconsistent; run validation/sync after upload.'
+    )
+  );
+}
+
+function logUploadTargetSection(environment, dataplaneUrl) {
+  logger.log('');
+  logger.log(sectionTitle('Target'));
+  logger.log(SEP);
+  logger.log(headerKeyValue('Environment:', environment));
+  logger.log(headerKeyValue('Dataplane:', dataplaneUrl));
+  logDataplanePipelineWarning();
+}
+
+/**
+ * Resolve dataplane auth and run upload preflight checks.
+ * @param {Function} resolveDataplaneAndAuth
+ * @param {Function} maybeRunVerboseServerValidation
+ * @param {string} systemKey
+ * @param {Object} options
+ * @param {Object} payload
+ * @returns {Promise<{ dataplaneUrl: string, authConfig: Object, environment: string }>}
+ */
+async function resolveUploadPublishContext(
+  resolveDataplaneAndAuth,
+  maybeRunVerboseServerValidation,
+  systemKey,
+  options,
+  payload
+) {
+  const resolved = await resolveDataplaneAndAuth(systemKey, {
+    silent: options.silentResolve === true
+  });
+  requireBearerForDataplanePipeline(resolved.authConfig);
+  if (options.force === true) {
+    logForcePublishWarning();
+  }
+  const controllerUrl = await resolveControllerUrl();
+  await assertDataplaneCliVersionCompatible(resolved.dataplaneUrl, { controllerUrl });
+  if (options.minimal !== true) {
+    logUploadTargetSection(resolved.environment, resolved.dataplaneUrl);
+  }
+  if (options.verbose) {
+    await maybeRunVerboseServerValidation(resolved.dataplaneUrl, resolved.authConfig, payload);
+  }
+  return resolved;
+}
+
+module.exports = {
+  logForcePublishWarning,
+  logUploadTargetSection,
+  resolveUploadPublishContext
+};

package/lib/commands/upload-sidecar-followups.js

@@ -0,0 +1,89 @@
+/**
+ * @fileoverview Upload sidecar follow-ups (governance packs, certification sync).
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const logger = require('../utils/logger');
+const { formatWarningLine } = require('../utils/cli-test-layout-chalk');
+const { unwrapPublicationResult, isPublicationUnchanged } = require('../utils/external-system-readiness-core');
+const { logUploadReadinessSummary } = require('../utils/external-system-readiness-display');
+const { maybeSyncSystemCertificationFromDataplane } = require('../certification/sync-system-certification');
+const { cliOptsSkipCertSync } = require('../certification/cli-cert-sync-skip');
+const { isQuietMinimalSync } = require('../utils/upload-sync-options');
+
+/**
+ * Sidecar follow-ups that are independent of pipeline publish hash (scenario packs, cert sync).
+ * @param {Object} ctx
+ * @returns {Promise<void>}
+ */
+async function runUploadSidecarFollowups(ctx) {
+  const { systemKey, options, payload, dataplaneUrl, authConfig } = ctx;
+  const quietSidecar = isQuietMinimalSync(options);
+  const dsKeys = (payload.dataSources || []).map((ds) => ds && ds.key).filter(Boolean);
+  try {
+    const { uploadGovernanceScenarioPacks } = require('../lifecycle/scenario-upload');
+    await uploadGovernanceScenarioPacks(systemKey, dataplaneUrl, authConfig, { silent: quietSidecar });
+  } catch (err) {
+    if (!quietSidecar) {
+      logger.log(formatWarningLine(`Governance scenario upload skipped: ${err.message}`));
+    }
+  }
+  await maybeSyncSystemCertificationFromDataplane({
+    label: 'upload',
+    noCertSync: cliOptsSkipCertSync(options),
+    systemKey,
+    dataplaneUrl,
+    authConfig,
+    datasourceKeys: dsKeys,
+    silent: quietSidecar
+  });
+}
+
+/**
+ * @param {Object} ctx
+ * @param {Function} maybeRunUploadProbe
+ * @returns {Promise<void>}
+ */
+async function handlePublicationAndFollowups(ctx, maybeRunUploadProbe) {
+  const {
+    systemKey,
+    options,
+    manifest,
+    payload,
+    environment,
+    dataplaneUrl,
+    authConfig,
+    rawRes
+  } = ctx;
+  const publication = unwrapPublicationResult(rawRes);
+  if (!publication) {
+    throw new Error(
+      'Unexpected response from dataplane upload: missing publication result (uploadId/system/datasources).'
+    );
+  }
+  if (isPublicationUnchanged(publication)) {
+    await runUploadSidecarFollowups({ systemKey, options, payload, dataplaneUrl, authConfig });
+    return;
+  }
+  logUploadReadinessSummary({
+    environment,
+    dataplaneUrl,
+    systemKey,
+    publication,
+    manifest,
+    minimal: !!options.minimal,
+    willProbe: !!options.probe
+  });
+  if (options.probe) {
+    await maybeRunUploadProbe(dataplaneUrl, systemKey, authConfig, options.probeTimeout);
+  }
+  await runUploadSidecarFollowups({ systemKey, options, payload, dataplaneUrl, authConfig });
+}
+
+module.exports = {
+  runUploadSidecarFollowups,
+  handlePublicationAndFollowups
+};

package/lib/commands/upload.js

@@ -21,10 +21,12 @@ const logger = require('../utils/logger');
 
 const SEP = chalk.gray('────────────────────────────────────────');
 const { resolveControllerUrl } = require('../utils/controller-url');
-const { getDeploymentAuth, requireBearerForDataplanePipeline } = require('../utils/token-manager');
+const { getDeploymentAuth } = require('../utils/token-manager');
 const { resolveDataplaneUrl } = require('../utils/dataplane-resolver');
 const { getIntegrationPath } = require('../utils/paths');
 const { pushCredentialSecrets } = require('../utils/credential-secrets-env');
+const { runWithCredentialPushSpinner } = require('../utils/credential-push-ui');
+const { runWithUploadPipelineSpinner } = require('../utils/upload-pipeline-ui');
 const {
   buildResolvedEnvMapForIntegration,
   resolveConfigurationValues,
@@ -34,21 +36,24 @@ const { validateExternalSystemComplete } = require('../validation/validate');
 const { displayValidationResults } = require('../validation/validate-display');
 const { generateControllerManifest } = require('../generator/external-controller-manifest');
 const {
-  uploadApplicationViaPipeline,
   validatePipelineConfig,
   testSystemViaPipeline
 } = require('../api/pipeline.api');
+const { uploadApplicationViaPipelineWithAuthRetry } = require('../utils/pipeline-upload-auth-retry');
 const { formatApiError } = require('../utils/api-error-handler');
-const { logDataplanePipelineWarning } = require('../utils/dataplane-pipeline-warning');
-const { unwrapPublicationResult, unwrapApiData } = require('../utils/external-system-readiness-core');
+const { unwrapPublicationResult, unwrapApiData, isPublicationUnchanged } = require('../utils/external-system-readiness-core');
 const {
-  logUploadReadinessSummary,
   logServerValidationWarnings,
   logProbeRuntimeBlock
 } = require('../utils/external-system-readiness-display');
-const { maybeSyncSystemCertificationFromDataplane } = require('../certification/sync-system-certification');
-const { cliOptsSkipCertSync } = require('../certification/cli-cert-sync-skip');
 const { maybeSyncOpenApiFilesForMcp } = require('./repair-openapi-sync');
+const { resolveUploadPublishContext } = require('./upload-publish-context');
+const { syncDeployJsonFromSources } = require('../external-system/sync-deploy-manifest');
+const { isQuietMinimalSync } = require('../utils/upload-sync-options');
+const {
+  runUploadSidecarFollowups,
+  handlePublicationAndFollowups
+} = require('./upload-sidecar-followups');
 
 /**
  * Validates system-key format (same as download).
@@ -70,13 +75,22 @@ function validateSystemKeyFormat(systemKey) {
  * @param {Object} manifest - Controller manifest from generateControllerManifest
  * @returns {Object} { version, application, dataSources, status: "draft" }
  */
-function buildUploadPayload(manifest) {
-  return {
+/**
+ * @param {Object} manifest - Controller manifest from generateControllerManifest
+ * @param {{ force?: boolean }} [opts]
+ * @returns {Object}
+ */
+function buildUploadPayload(manifest, opts = {}) {
+  const payload = {
     version: manifest.version || '1.0.0',
     application: manifest.system,
     dataSources: manifest.dataSources || [],
     status: 'draft'
   };
+  if (opts.force === true) {
+    payload.force = true;
+  }
+  return payload;
 }
 
 /**
@@ -110,11 +124,15 @@ async function resolveDataplaneAndAuth(systemKey, opts = {}) {
  * @param {Object} payload - { version, application, dataSources, status: "draft" }
  * @returns {Promise<Object>} Raw API response envelope
  */
-async function runUploadValidatePublish(dataplaneUrl, authConfig, payload) {
-  const res = await uploadApplicationViaPipeline(dataplaneUrl, authConfig, payload);
+async function runUploadValidatePublish(dataplaneUrl, authConfig, payload, systemKey) {
+  const res = await uploadApplicationViaPipelineWithAuthRetry(
+    dataplaneUrl,
+    authConfig,
+    payload,
+    systemKey
+  );
   if (res?.success === false) {
-    const msg = formatApiError(res, dataplaneUrl);
-    throw new Error(msg);
+    throw new Error(formatApiError(res, dataplaneUrl));
   }
   return res;
 }
@@ -170,13 +188,17 @@ async function resolvePayloadConfiguration(systemKey, payload) {
  * @param {string} systemKey - System key
  * @param {Object} payload - Upload payload
  */
-async function pushAndLogCredentialSecrets(dataplaneUrl, authConfig, systemKey, payload) {
+async function pushAndLogCredentialSecrets(dataplaneUrl, authConfig, systemKey, payload, uiOpts = {}) {
   const envFilePath = path.join(getIntegrationPath(systemKey), '.env');
-  const pushResult = await pushCredentialSecrets(dataplaneUrl, authConfig, {
-    envFilePath,
-    appName: systemKey,
-    payload
-  });
+  const pushResult = await runWithCredentialPushSpinner(
+    () =>
+      pushCredentialSecrets(dataplaneUrl, authConfig, {
+        envFilePath,
+        appName: systemKey,
+        payload
+      }),
+    { systemKey, json: uiOpts.json === true }
+  );
   if (pushResult.pushed > 0) {
     const keyList = pushResult.keys?.length ? ` (${pushResult.keys.join(', ')})` : '';
     logger.log(
@@ -262,18 +284,22 @@ async function maybeRunUploadProbe(dataplaneUrl, systemKey, authConfig, probeTim
 
 /**
  * Local validation, manifest, payload, and configuration resolution.
- * Does not write *-deploy.json; run `aifabrix json <systemKey>` to regenerate the deployment manifest from sources.
+ * Regenerates integration/<systemKey>/<systemKey>-deploy.json from sources (same as `aifabrix json`).
  *
  * @param {string} systemKey
  * @param {{ dryRun?: boolean }} [_opts] - Reserved for callers (e.g. dry-run); manifest is always built from current disk files
  * @returns {Promise<{ manifest: Object, payload: Object }>}
  */
-async function buildValidatedUploadManifestPayload(systemKey, _opts = {}) {
+async function buildValidatedUploadManifestPayload(systemKey, opts = {}) {
+  const quiet = opts.minimal === true || opts.quiet === true;
   const validationResult = await validateExternalSystemComplete(systemKey, { type: 'external' });
   throwIfValidationFailed(validationResult);
-  logger.log(formatSuccessLine('Local validation passed'));
+  if (!quiet) {
+    logger.log(formatSuccessLine('Local validation passed'));
+  }
+  await syncDeployJsonFromSources(systemKey, { quiet });
   const manifest = await generateControllerManifest(systemKey, { type: 'external' });
-  const payload = buildUploadPayload(manifest);
+  const payload = buildUploadPayload(manifest, { force: opts.force === true });
   await resolvePayloadConfiguration(systemKey, payload);
   return { manifest, payload };
 }
@@ -304,58 +330,6 @@ async function logAndSyncLocalOpenApiForUpload(systemKey, manifest) {
   }
 }
 
-/**
- * @param {Object} ctx
- * @param {string} ctx.systemKey
- * @param {Object} ctx.options
- * @param {Object} ctx.manifest
- * @param {Object} ctx.payload
- * @param {string} ctx.environment
- * @param {string} ctx.dataplaneUrl
- * @param {Object} ctx.authConfig
- * @param {Object} ctx.rawRes
- * @returns {Promise<void>}
- */
-async function handlePublicationAndFollowups(ctx) {
-  const {
-    systemKey,
-    options,
-    manifest,
-    payload,
-    environment,
-    dataplaneUrl,
-    authConfig,
-    rawRes
-  } = ctx;
-  const publication = unwrapPublicationResult(rawRes);
-  if (!publication) {
-    throw new Error(
-      'Unexpected response from dataplane upload: missing publication result (uploadId/system/datasources).'
-    );
-  }
-  logUploadReadinessSummary({
-    environment,
-    dataplaneUrl,
-    systemKey,
-    publication,
-    manifest,
-    minimal: !!options.minimal,
-    willProbe: !!options.probe
-  });
-  if (options.probe) {
-    await maybeRunUploadProbe(dataplaneUrl, systemKey, authConfig, options.probeTimeout);
-  }
-  const dsKeys = (payload.dataSources || []).map((ds) => ds && ds.key).filter(Boolean);
-  await maybeSyncSystemCertificationFromDataplane({
-    label: 'upload',
-    noCertSync: cliOptsSkipCertSync(options),
-    systemKey,
-    dataplaneUrl,
-    authConfig,
-    datasourceKeys: dsKeys
-  });
-}
-
 /**
  * Upload path after dry-run check.
  * @param {string} systemKey
@@ -364,31 +338,55 @@ async function handlePublicationAndFollowups(ctx) {
  * @param {Object} payload
  */
 async function runUploadPublishAndSummary(systemKey, options, manifest, payload) {
-  const { dataplaneUrl, authConfig, environment } = await resolveDataplaneAndAuth(systemKey);
-  requireBearerForDataplanePipeline(authConfig);
-  logger.log('');
-  logger.log(sectionTitle('Target'));
-  logger.log(SEP);
-  logger.log(headerKeyValue('Environment:', environment));
-  logger.log(headerKeyValue('Dataplane:', dataplaneUrl));
-  logDataplanePipelineWarning();
-  if (options.verbose) {
-    await maybeRunVerboseServerValidation(dataplaneUrl, authConfig, payload);
-  }
-  await pushAndLogCredentialSecrets(dataplaneUrl, authConfig, systemKey, payload);
-
-  const rawRes = await runUploadValidatePublish(dataplaneUrl, authConfig, payload);
-  await logAndSyncLocalOpenApiForUpload(systemKey, manifest);
-  await handlePublicationAndFollowups({
+  const { dataplaneUrl, authConfig, environment } = await resolveUploadPublishContext(
+    resolveDataplaneAndAuth,
+    maybeRunVerboseServerValidation,
     systemKey,
     options,
-    manifest,
-    payload,
-    environment,
-    dataplaneUrl,
-    authConfig,
-    rawRes
+    payload
+  );
+  const rawRes = await runWithUploadPipelineSpinner(
+    async() => runUploadValidatePublish(dataplaneUrl, authConfig, payload, systemKey),
+    {
+      systemKey,
+      json: options.json === true,
+      syncMode: options.syncMode === true,
+      suppressPipelineSpinner: options.suppressPipelineSpinner === true
+    }
+  );
+  const publication = unwrapPublicationResult(rawRes);
+  if (isPublicationUnchanged(publication)) {
+    if (!isQuietMinimalSync(options)) {
+      logger.log(formatSuccessLine('Config unchanged on dataplane — upload skipped.'));
+    }
+    await runUploadSidecarFollowups({
+      systemKey,
+      options,
+      payload,
+      dataplaneUrl,
+      authConfig
+    });
+    return { dataplaneUrl, authConfig, environment, uploadSkipped: true };
+  }
+
+  await pushAndLogCredentialSecrets(dataplaneUrl, authConfig, systemKey, payload, {
+    json: options.json === true
   });
+  await logAndSyncLocalOpenApiForUpload(systemKey, manifest);
+  await handlePublicationAndFollowups(
+    {
+      systemKey,
+      options,
+      manifest,
+      payload,
+      environment,
+      dataplaneUrl,
+      authConfig,
+      rawRes
+    },
+    maybeRunUploadProbe
+  );
+  return { dataplaneUrl, authConfig, environment, uploadSkipped: false };
 }
 
 /**
@@ -400,26 +398,31 @@ async function runUploadPublishAndSummary(systemKey, options, manifest, payload)
  * @param {boolean} [options.probe] - POST validation/run after publish
  * @param {number} [options.probeTimeout] - Probe timeout ms (default 120000)
  * @param {boolean} [options.minimal] - Short summary only
- * @returns {Promise<void>}
+ * @returns {Promise<{ dataplaneUrl: string, authConfig: Object, environment: string }|void>}
  * @throws {Error} If validation or API calls fail
  */
 async function uploadExternalSystem(systemKey, options = {}) {
   validateSystemKeyFormat(systemKey);
-  logger.log('');
-  logger.log(sectionTitle('Upload external system'));
-  logger.log(SEP);
-  logger.log(headerKeyValue('System:', systemKey));
+  const minimal = options.minimal === true;
+  if (!minimal) {
+    logger.log('');
+    logger.log(sectionTitle('Upload external system'));
+    logger.log(SEP);
+    logger.log(headerKeyValue('System:', systemKey));
+  }
   const { manifest, payload } = await buildValidatedUploadManifestPayload(systemKey, {
-    dryRun: !!options.dryRun
+    dryRun: !!options.dryRun,
+    force: options.force === true,
+    minimal
   });
   if (options.dryRun) {
     logger.log(chalk.yellow('Dry run: would upload payload (no API calls).'));
     logger.log(
       chalk.gray(`  System: ${manifest.key}, version: ${payload.version}, datasources: ${payload.dataSources.length}`)
     );
-    return;
+    return undefined;
   }
-  await runUploadPublishAndSummary(systemKey, options, manifest, payload);
+  return runUploadPublishAndSummary(systemKey, options, manifest, payload);
 }
 
 module.exports = {

package/lib/commands/verify-governance-combined-external.js

@@ -0,0 +1,61 @@
+/**
+ * @fileoverview verify-governance — sync, generate, verify (424.0 dataplane-first).
+ */
+
+'use strict';
+
+const { runGovernanceGenerateForExternalSystem } = require('./governance-generate-external');
+const { runVerifyGovernanceForExternalSystem } = require('./verify-governance-external');
+const {
+  buildSubjectsFromOptions,
+  resolveGovernanceIntegrationContext
+} = require('./governance-cli-shared');
+
+/**
+ * @async
+ * @param {string} systemKey
+ * @param {Object} options
+ * @returns {Promise<Object>}
+ */
+async function runVerifyGovernanceCombined(systemKey, options = {}) {
+  if (options.pack) {
+    return runVerifyGovernanceForExternalSystem(systemKey, options);
+  }
+
+  const subjects = buildSubjectsFromOptions(options);
+  if (subjects.length === 0) {
+    throw new Error(
+      'At least one --subject-user-id or --subject-email is required for verify-governance'
+    );
+  }
+
+  const { resolvedKey, authConfig, dataplaneUrl } = await resolveGovernanceIntegrationContext(
+    systemKey,
+    options
+  );
+
+  const pipelineOpts = {
+    ...options,
+    noSync: true,
+    authConfig,
+    dataplaneUrl
+  };
+
+  const generateResult = await runGovernanceGenerateForExternalSystem(
+    resolvedKey,
+    pipelineOpts
+  );
+
+  const verifyResult = await runVerifyGovernanceForExternalSystem(resolvedKey, pipelineOpts);
+
+  return {
+    ...verifyResult,
+    command: 'verify-governance',
+    generated: generateResult.generated,
+    generateResults: generateResult.results
+  };
+}
+
+module.exports = {
+  runVerifyGovernanceCombined
+};

package/lib/commands/verify-governance-command-action.js

@@ -0,0 +1,64 @@
+/**
+ * @fileoverview CLI action for `aifabrix verify-governance` (plan 150.0 / 424.0).
+ */
+
+'use strict';
+
+const logger = require('../utils/logger');
+const { handleCommandError } = require('../utils/cli-utils');
+const { runVerifyGovernanceCombined } = require('./verify-governance-combined-external');
+const { normalizeTestGovernanceCliOptions } = require('./test-governance-cli-options');
+const { displayVerifyGovernanceTTY } = require('../lifecycle/report-display');
+const { exitCodeFromVerdict } = require('../lifecycle/exit-codes');
+const { buildJsonEnvelope } = require('../lifecycle/json-envelope');
+const { formatSuccessLine } = require('../utils/cli-test-layout-chalk');
+
+/**
+ * @param {Object} options
+ * @param {import('commander').Command} [cmd]
+ * @returns {Object}
+ */
+function normalizeVerifyGovernanceCliOptions(options, cmd) {
+  const base = normalizeTestGovernanceCliOptions(options, cmd);
+  return {
+    ...base,
+    certificationProfile: options.certificationProfile || 'bronze',
+    subjectUserIds: options.subjectUserId || [],
+    subjectEmails: options.subjectEmail || [],
+    maxSamplesPerPartition: options.maxSamplesPerPartition
+  };
+}
+
+async function runVerifyGovernanceCommandAction(appName, options, cmd) {
+  const opts = normalizeVerifyGovernanceCliOptions(options, cmd);
+  const result = await runVerifyGovernanceCombined(appName, opts);
+
+  if (opts.json) {
+    logger.log(JSON.stringify(buildJsonEnvelope(result, opts.verbose), null, 2));
+  } else {
+    if (opts.verbose && !opts.pack && typeof result.generated === 'number' && result.generated > 0) {
+      logger.log(formatSuccessLine(`Generated packs for ${result.generated} datasource(s)`));
+    }
+    displayVerifyGovernanceTTY(result.systemKey, result, { details: opts.verbose });
+  }
+
+  const exitCode = exitCodeFromVerdict(result.verdict);
+  if (exitCode !== 0) {
+    process.exit(exitCode);
+  }
+}
+
+async function verifyGovernanceCommandHandler(appName, options, cmd) {
+  try {
+    await runVerifyGovernanceCommandAction(appName, options, cmd);
+  } catch (error) {
+    handleCommandError(error, 'verify-governance');
+    process.exit(3);
+  }
+}
+
+module.exports = {
+  runVerifyGovernanceCommandAction,
+  verifyGovernanceCommandHandler,
+  normalizeVerifyGovernanceCliOptions
+};