@aifabrix/builder 2.60.0 → 2.62.0

package/lib/commands/up-common.js

@@ -331,12 +331,16 @@ const DATAPLANE_BUILDER_APP_KEYS = Object.freeze(['dataplane']);
 /** Builder API platform app key (154.2). */
 const BUILDER_API_APP_KEYS = Object.freeze(['builder-api']);
 
+/** Optional integration server platform app key. */
+const INTEGRATION_SERVER_APP_KEYS = Object.freeze(['integration-server']);
+
 /** Builder dirs removed on up-platform / setup force (includes Celery companions). */
 const PLATFORM_BUILDER_CLEAN_APP_KEYS = Object.freeze([
   'keycloak',
   'miso-controller',
   ...DATAPLANE_BUILDER_APP_KEYS,
-  ...BUILDER_API_APP_KEYS
+  ...BUILDER_API_APP_KEYS,
+  ...INTEGRATION_SERVER_APP_KEYS
 ]);
 
 /**
@@ -375,6 +379,15 @@ async function ensureBuilderApiAppsFromTemplate(opts = {}) {
   return anyCopied;
 }
 
+/**
+ * Ensure the optional integration server manifest exists under the system builder root.
+ * @param {object} [opts]
+ * @returns {Promise<boolean>}
+ */
+async function ensureIntegrationServerAppsFromTemplate(opts = {}) {
+  return ensureAppFromTemplate(INTEGRATION_SERVER_APP_KEYS[0], opts);
+}
+
 /**
  * Merge builder/packages `application.yaml` into `~/.aifabrix/urls.local.yaml`.
  * Same scan as the end of {@link prepareUrlsLocalRegistryForUpPlatform}; call after platform app
@@ -410,10 +423,12 @@ module.exports = {
   applyUpPlatformForceConfig,
   cleanBuilderAppDirs,
   BUILDER_API_APP_KEYS,
+  INTEGRATION_SERVER_APP_KEYS,
   DATAPLANE_BUILDER_APP_KEYS,
   PLATFORM_BUILDER_CLEAN_APP_KEYS,
   ensureAppFromTemplate,
   ensureBuilderApiAppsFromTemplate,
+  ensureIntegrationServerAppsFromTemplate,
   ensureDataplaneAppsFromTemplate,
   patchEnvOutputPathForDeployOnly,
   validateEnvOutputPathFolderOrNull,

package/lib/commands/up-dataplane-bootstrap.js

@@ -19,8 +19,12 @@ const {
   extractAppConfiguration
 } = require('../utils/app-register-config');
 const { buildRegistrationData, saveLocalCredentials, applyLocalhostPortAdjustment } = require('../app/register');
-const { loadDataplanePipelineCredentialsFromSecretsFile } = require('./up-dataplane-credentials');
 const { formatSuccessLine } = require('../utils/cli-test-layout-chalk');
+const { getToken } = require('../api/auth.api');
+const { updateApplicationSelfStatus } = require('../api/applications.api');
+const { tokenFrom } = require('../app/run-parameter-sync');
+const { resolveDeclarativeShowUrlsForApp } = require('../core/secrets-env-declarative-expand');
+const pathsUtil = require('../utils/paths');
 
 const ENCRYPTION_VAULT_KEYS = [
   'miso-controller-secrets-encryptionKeyVault',
@@ -87,6 +91,68 @@ async function canUseBootstrapProvision(options = {}) {
   return Boolean(encryptionKey);
 }
 
+/**
+ * Persist canonical declarative URLs before Dataplane takes its bootstrap snapshot.
+ * @param {string} controllerUrl
+ * @param {string} environmentKey
+ * @param {{clientId: string, clientSecret: string}} credentials
+ * @param {number} containerPort
+ * @returns {Promise<void>}
+ */
+async function syncDataplaneApplicationUrls(controllerUrl, environmentKey, credentials, containerPort) {
+  const appPath = pathsUtil.getBuilderPath('dataplane');
+  const variablesPath = pathsUtil.resolveApplicationConfigPath(appPath);
+  const urls = await resolveDeclarativeShowUrlsForApp('dataplane', appPath, variablesPath, 'docker');
+  if (!urls) {
+    throw new Error('Could not resolve canonical Dataplane public and internal URLs');
+  }
+  const tokenResponse = await getToken(credentials.clientId, credentials.clientSecret, controllerUrl);
+  const response = await updateApplicationSelfStatus(
+    controllerUrl,
+    environmentKey,
+    { type: 'client-token', token: tokenFrom(tokenResponse) },
+    { status: 'deploying', url: urls.publicUrl, internalUrl: urls.internalUrl, port: containerPort }
+  );
+  if (!response?.success) {
+    throw new Error('Controller rejected canonical Dataplane public and internal URLs');
+  }
+}
+
+async function handleBootstrapResponse(responseData, controllerUrl, environmentKey, containerPort) {
+  if (responseData.credentials && responseData.credentials.clientSecret) {
+    await saveLocalCredentials(responseData, controllerUrl);
+    await syncDataplaneApplicationUrls(
+      controllerUrl,
+      environmentKey,
+      responseData.credentials,
+      containerPort
+    );
+    logger.log(formatSuccessLine('Dataplane bootstrap credentials saved locally'));
+    const { waitForDataplaneCredentials } = require('./up-dataplane-credentials');
+    if (!(await waitForDataplaneCredentials(controllerUrl))) {
+      throw new Error(
+        'Dataplane bootstrap credentials were issued but the controller did not accept them in time. ' +
+        'The deploy that follows gets one attempt, so it would fail with a less specific error. ' +
+        'Retry the command; if it persists, check the controller is serving /api/v1/auth/token.'
+      );
+    }
+    return;
+  }
+  if (responseData.status === 'already_provisioned') {
+    logger.log(formatSuccessLine('Dataplane already provisioned (bootstrap idempotent)'));
+    const { credentialsAcceptToken } = require('./up-dataplane-credentials');
+    if (await credentialsAcceptToken(controllerUrl)) return;
+    throw new Error(
+      'Dataplane is registered on the controller but local pipeline credentials are missing or ' +
+        'not accepted by it. Run: aifabrix login && aifabrix app rotate-secret dataplane'
+    );
+  }
+  throw new Error(
+    'Dataplane bootstrap returned no credentials and no "already_provisioned" status ' +
+      `(status: ${responseData.status || 'none'}). The controller may be a version this CLI does not support.`
+  );
+}
+
 /**
  * Register dataplane via bootstrap endpoint (idempotent).
  * @async
@@ -107,6 +173,7 @@ async function bootstrapDataplaneApplication(options, controllerUrl, environment
 
   const { variables } = await loadVariablesYaml('dataplane');
   const appConfig = await extractAppConfiguration(variables, 'dataplane', options);
+  const containerPort = appConfig.port;
   await applyLocalhostPortAdjustment(appConfig, controllerUrl, options);
 
   const imageOverride = options.image || buildDataplaneImageRefFn(options);
@@ -126,24 +193,13 @@ async function bootstrapDataplaneApplication(options, controllerUrl, environment
     registrationData
   );
 
-  if (responseData.credentials && responseData.credentials.clientSecret) {
-    await saveLocalCredentials(responseData, controllerUrl);
-    logger.log(formatSuccessLine('Dataplane bootstrap credentials saved locally'));
-  } else if (responseData.status === 'already_provisioned') {
-    logger.log(formatSuccessLine('Dataplane already provisioned (bootstrap idempotent)'));
-    const localCredentials = await loadDataplanePipelineCredentialsFromSecretsFile();
-    if (!localCredentials) {
-      throw new Error(
-        'Dataplane is registered on the controller but pipeline credentials are missing locally. ' +
-          'Run fresh setup with volume wipe, or: aifabrix login && aifabrix app rotate-secret dataplane'
-      );
-    }
-  }
+  await handleBootstrapResponse(responseData, controllerUrl, environmentKey, containerPort);
 }
 
 module.exports = {
   canUseBootstrapProvision,
   resolveBootstrapEncryptionKey,
   bootstrapDataplaneApplication,
+  syncDataplaneApplicationUrls,
   pickBootstrapEncryptionKeyFromSecrets
 };

package/lib/commands/up-dataplane-credentials.js

@@ -193,8 +193,29 @@ async function alignDataplaneCredentialsAfterDeploy(options, controllerUrl) {
   }
 }
 
+/**
+ * Wait until a freshly bootstrapped pair is accepted, before anything depends on it.
+ *
+ * The Controller can answer the bootstrap call before the pipeline client is usable, and
+ * the deploy that follows gets one attempt against one URL: validateDeployment retries only
+ * on a 5xx, so a 401 from a client that is not published yet ends the install. The align
+ * step already tolerates this, with 15 attempts across three bases, but it runs after the
+ * deploy — the forgiving check after the unforgiving one.
+ *
+ * @async
+ * @param {string} controllerUrl
+ * @param {Object} [retry] - Overrides for the bootstrap probe budget
+ * @returns {Promise<boolean>} True once the pair is accepted
+ */
+async function waitForDataplaneCredentials(controllerUrl, retry = {}) {
+  const { maxAttempts, delayMs } = resolveCredentialProbeRetry({ useBootstrap: true, credentialProbeRetry: retry });
+  return credentialsAcceptTokenWithRetry(controllerUrl, maxAttempts, delayMs);
+}
+
 module.exports = {
   alignDataplaneCredentialsAfterDeploy,
+  waitForDataplaneCredentials,
+  credentialsAcceptTokenWithRetry,
   probeDataplaneClientToken,
   credentialsAcceptToken,
   loadDataplanePipelineCredentialsFromSecretsFile,

package/lib/commands/up-dataplane.js

@@ -227,6 +227,9 @@ async function deployDataplaneToController(options) {
     image: imageOverride,
     registryMode: options.registryMode,
     envKey: options.envKey ? String(options.envKey).trim().toLowerCase() : undefined,
+    // The bootstrap path has no device login, so it sets client-credentials explicitly.
+    // Dropping it here left the deploy in 'auto' mode.
+    deploymentAuth: options.deploymentAuth,
     // Guided up-platform already shows a top-level spinner for the dataplane step.
     // Avoid nested deploy polling spinners/logs.
     silentPoll: options.platformInstall === true
@@ -266,7 +269,8 @@ function emitDataplaneManifestLineFromBuilder() {
 }
 
 function computeDataplanePortOverride(runEnvKey, developerIdNum) {
-  if (runEnvKey !== 'dev' && runEnvKey !== 'tst') {
+  const { isScopedRunEnvironmentKey } = require('../utils/environment-scoped-resources');
+  if (!isScopedRunEnvironmentKey(runEnvKey)) {
     return undefined;
   }
   return dataplaneDockerHostPort(3001, developerIdNum, runEnvKey);
@@ -355,6 +359,37 @@ async function provisionAndDeployDataplane(options, environmentKey, controllerUr
  * @returns {Promise<void>}
  * @throws {Error} If infra not up, controller unavailable, not logged in, environment not dev, or any step fails
  */
+/**
+ * Run options for the local dataplane container.
+ *
+ * A guided platform install already shows one spinner for the whole dataplane step, so it
+ * either forwards its own progress callback or asks for silence; a standalone run prints
+ * normally.
+ *
+ * @param {Object} options - Command options
+ * @param {string} runEnvKey - Resolved run environment
+ * @param {number|undefined} portOverride - Resolved port override
+ * @returns {Object} Options for app.runApp
+ */
+function buildDataplaneRunOptions(options, runEnvKey, portOverride) {
+  const runAppOpts = {
+    skipEnvOutputPath: true,
+    env: runEnvKey,
+    port: portOverride,
+    registry: options.registry || undefined,
+    base: options.base !== false,
+    skipManifestMetadataLine: true
+  };
+  if (options.platformInstall === true) {
+    if (typeof options.onPollText === 'function') {
+      runAppOpts.onPollText = options.onPollText;
+    } else {
+      runAppOpts.silentPoll = true;
+    }
+  }
+  return runAppOpts;
+}
+
 async function handleUpDataplane(options = {}) {
   if (options.platformInstall !== true) {
     logger.log(chalk.blue('Starting up-dataplane (register/rotate, deploy, then run dataplane locally)...\n'));
@@ -386,23 +421,7 @@ async function handleUpDataplane(options = {}) {
   );
 
   const { runEnvKey, portOverride } = await computeDataplaneRunOverrides(options);
-
-  const runAppOpts = {
-    skipEnvOutputPath: true,
-    env: runEnvKey,
-    port: portOverride,
-    registry: options.registry || undefined,
-    base: options.base !== false,
-    skipManifestMetadataLine: true
-  };
-  if (options.platformInstall === true) {
-    if (typeof options.onPollText === 'function') {
-      runAppOpts.onPollText = options.onPollText;
-    } else {
-      runAppOpts.silentPoll = true;
-    }
-  }
-  await app.runApp('dataplane', runAppOpts);
+  await app.runApp('dataplane', buildDataplaneRunOptions(options, runEnvKey, portOverride));
 
   if (options.platformInstall !== true) {
     logger.log(formatSuccessParagraph('up-dataplane complete. Dataplane is registered, deployed in dev, and running locally.'));

package/lib/commands/up-integration-server.js

@@ -0,0 +1,112 @@
+/**
+ * Install the optional Integration Server as a miso-only platform application.
+ *
+ * @fileoverview Integration Server setup phase
+ */
+
+'use strict';
+
+const app = require('../app');
+const config = require('../core/config');
+const { generateEnvFile } = require('../core/secrets');
+const pathsUtil = require('../utils/paths');
+const { loadConfigFile } = require('../utils/config-format');
+const { resolveDockerImageRef } = require('../utils/resolve-docker-image-ref');
+const { callBootstrapApi } = require('../utils/app-bootstrap-api');
+const { loadVariablesYaml, extractAppConfiguration } = require('../utils/app-register-config');
+const { buildRegistrationData, saveLocalCredentials, applyLocalhostPortAdjustment } = require('../app/register');
+const { publishedHostPort, parseDeveloperIdNum } = require('../utils/declarative-url-ports');
+const { resolveBootstrapEncryptionKey } = require('./up-dataplane-bootstrap');
+const { resolveUpBuilderApiControllerContext } = require('./up-builder-api-provision');
+const {
+  ensureIntegrationServerAppsFromTemplate,
+  refreshUrlsLocalRegistryForCurrentProject
+} = require('./up-common');
+const { resolveLocalOnboardingApps } = require('../core/local-env-overrides');
+
+const APP_KEY = 'integration-server';
+const ENVIRONMENT_KEY = 'miso';
+
+async function isIntegrationServerSelected() {
+  const cfg = await config.getConfig();
+  return resolveLocalOnboardingApps(cfg).includes(APP_KEY);
+}
+
+function resolveIntegrationServerImageRef(options = {}) {
+  const appPath = pathsUtil.getBuilderPath(APP_KEY);
+  const manifestPath = pathsUtil.resolveApplicationConfigPath(appPath);
+  const variables = loadConfigFile(manifestPath) || {};
+  const { imageName, imageTag } = resolveDockerImageRef(APP_KEY, variables, {
+    registry: options.registry
+  });
+  return `${imageName}:${imageTag}`;
+}
+
+async function bootstrapIntegrationServer(options, controllerUrl) {
+  const encryptionKey = await resolveBootstrapEncryptionKey();
+  if (!encryptionKey) {
+    throw new Error('Bootstrap encryption key is unavailable for Integration Server installation.');
+  }
+  const { variables } = await loadVariablesYaml(APP_KEY);
+  const appConfig = await extractAppConfiguration(variables, APP_KEY, options);
+  await applyLocalhostPortAdjustment(appConfig, controllerUrl, options);
+  const image = options.image || resolveIntegrationServerImageRef(options);
+  const registrationData = buildRegistrationData(appConfig, {
+    image,
+    imageOverride: image,
+    registryMode: options.registryMode,
+    envKey: ENVIRONMENT_KEY
+  });
+  const response = await callBootstrapApi(
+    controllerUrl,
+    encryptionKey,
+    ENVIRONMENT_KEY,
+    registrationData
+  );
+  if (!response.credentials || !response.credentials.clientSecret) {
+    throw new Error('Integration Server bootstrap did not return its existing credentials.');
+  }
+  await saveLocalCredentials(response, controllerUrl);
+}
+
+async function runIntegrationServer(options) {
+  await generateEnvFile(APP_KEY, null, 'local', true);
+  const developerId = parseDeveloperIdNum(await config.getDeveloperId());
+  await app.runApp(APP_KEY, {
+    env: ENVIRONMENT_KEY,
+    registry: options.registry,
+    registryMode: options.registryMode,
+    image: options.image,
+    port: publishedHostPort(3006, developerId),
+    skipEnvOutputPath: true,
+    skipInfraCheck: true,
+    skipManifestMetadataLine: true,
+    silentPoll: true
+  });
+}
+
+async function handleUpIntegrationServer(options = {}) {
+  await ensureIntegrationServerAppsFromTemplate({ silent: true });
+  refreshUrlsLocalRegistryForCurrentProject();
+  const context = await resolveUpBuilderApiControllerContext(options, ENVIRONMENT_KEY);
+  if (!context.useBootstrap) {
+    throw new Error('Integration Server installation requires local bootstrap provisioning.');
+  }
+  await bootstrapIntegrationServer(options, context.controllerUrl);
+  const image = options.image || resolveIntegrationServerImageRef(options);
+  await app.deployApp(APP_KEY, {
+    image,
+    imageOverride: image,
+    registryMode: options.registryMode,
+    envKey: ENVIRONMENT_KEY,
+    deploymentAuth: 'client-credentials',
+    silentPoll: true
+  });
+  await runIntegrationServer({ ...options, image });
+}
+
+module.exports = {
+  handleUpIntegrationServer,
+  isIntegrationServerSelected,
+  resolveIntegrationServerImageRef
+};

package/lib/commands/verify-operations-skip-e2e.js

@@ -105,7 +105,8 @@ function shouldSkipE2eForStaticPackage(systemKey, options) {
     if (storage.length === 0) {
       return false;
     }
-    return storage.every((ds) => String(ds.syncBridge || 'disabled').toLowerCase() === 'disabled');
+    return storage.every((ds) => !isKeyedDocumentDatasource(ds) &&
+      String(ds.syncBridge || 'disabled').toLowerCase() === 'disabled');
   } catch {
     return false;
   }
@@ -124,7 +125,30 @@ function shouldSkipOperationsE2e(systemKey, options) {
     || shouldSkipE2eForStaticPackage(systemKey, options);
 }
 
+/** @param {Object} ds @returns {boolean} */
+function isKeyedDocumentDatasource(ds) {
+  const ops = ds?.execution?.cip?.operations;
+  return ds?.entityType === 'documentStorage' && !ops?.list &&
+    !ds.openapi?.operations?.list && !ds.capabilities?.some(cap => (cap.key || cap) === 'list') &&
+    Array.isArray(ops?.get?.steps) && ops.get.steps.some(step => step?.output?.mode === 'documents');
+}
+
+/** @param {string} systemKey @param {Object} options @returns {string[]} */
+function keyedDocumentDatasourceKeys(systemKey, options) {
+  if (options.skipE2e === true || options.minimal === true) return [];
+  try {
+    const { resolveAppWorkspace } = require('../utils/app-path-resolve');
+    const { appPath } = resolveAppWorkspace(systemKey, options);
+    const storage = loadStorageDatasources(appPath);
+    return storage.length && storage.every(isKeyedDocumentDatasource) ? storage.map(ds => ds.key) : [];
+  } catch {
+    return [];
+  }
+}
+
 module.exports = {
+  isKeyedDocumentDatasource,
+  keyedDocumentDatasourceKeys,
   shouldSkipOperationsE2e,
   shouldSkipE2eForRoleAssistant,
   shouldSkipE2eForStaticPackage

package/lib/commands/verify-operations-steps.js

@@ -4,6 +4,9 @@
 
 'use strict';
 
+const { keyedDocumentDatasourceKeys } = require('./verify-operations-skip-e2e');
+const { hasAllKeyedDocumentPersistence } = require('../utils/datasource-binary-evidence');
+
 const { syncLocalIfRequested } = require('./test-e2e-external');
 const { cliOptsSkipPublish } = require('../utils/cli-sync-options');
 const { buildOperationsVerifyStepPlan } = require('./verify-operations-step-plan');
@@ -99,10 +102,19 @@ async function runOperationsVerifySteps(systemKey, options) {
     }
   }
 
+  const keyedKeys = keyedDocumentDatasourceKeys(systemKey, options);
   const e2eLabel = STEP_PROGRESS.e2e;
   stepResults.e2e = await runWithVerifyStepProgress(
     e2eLabel,
-    async(progress) => runVerifyStep(() => plan.e2e.run(progress), plan.e2e.ok),
+    async(progress) => {
+      const result = await plan.e2e.run(progress);
+      if (keyedKeys.length) {
+        stepResults.keyedDocumentGet = true;
+        stepResults.documentPersistence = hasAllKeyedDocumentPersistence(result, keyedKeys);
+        return plan.e2e.ok(result) && stepResults.documentPersistence;
+      }
+      return plan.e2e.ok(result);
+    },
     options
   );
   return { stepResults, hardFailure: !stepResults.e2e, stoppedEarly: false };

package/lib/commands/wizard-config-normalizer.js

@@ -4,7 +4,11 @@
  * @version 2.0.0
  */
 
-const ENTITY_TYPE_FALLBACK = 'record-storage';
+const { repairKeyedDocument } = require('./repair-datasource-keyed-document');
+
+const { repairLegacyDatasource } = require('./repair-datasource-legacy');
+
+const ENTITY_TYPE_FALLBACK = 'recordStorage';
 const VALID_ENTITY_TYPES = new Set([
   'document-storage',
   'documentStorage',
@@ -14,6 +18,7 @@ const VALID_ENTITY_TYPES = new Set([
   'recordStorage',
   'message-service',
   'messageService',
+  'serviceContract',
   'none'
 ]);
 const VALID_PORTAL_FIELDS = new Set(['text', 'textarea', 'select', 'json', 'boolean', 'number']);
@@ -56,7 +61,7 @@ function normalizeCipOutputModes(datasourceConfig) {
   for (const operation of Object.values(operations)) {
     if (!operation || !Array.isArray(operation.steps)) continue;
     for (const step of operation.steps) {
-      if (step?.output?.mode && step.output.mode !== 'records') {
+      if (step?.output?.mode && !['records', 'documents'].includes(step.output.mode)) {
         step.output.mode = 'records';
       }
     }
@@ -80,15 +85,17 @@ function normalizeWizardTestPayload(datasourceConfig) {
  * @param {Object} datasourceConfig - Datasource config
  * @returns {Object} Normalized config
  */
-function normalizeDatasourceConfig(datasourceConfig) {
+function normalizeDatasourceConfig(datasourceConfig, openapiSpec) {
   if (!datasourceConfig || typeof datasourceConfig !== 'object') {
     return datasourceConfig;
   }
   if (datasourceConfig.entityType && !VALID_ENTITY_TYPES.has(datasourceConfig.entityType)) {
     datasourceConfig.entityType = ENTITY_TYPE_FALLBACK;
   }
+  repairLegacyDatasource(datasourceConfig);
   normalizePortalInput(datasourceConfig);
   normalizeCipOutputModes(datasourceConfig);
+  repairKeyedDocument(datasourceConfig, { openapiSpec });
   normalizeWizardTestPayload(datasourceConfig);
   return datasourceConfig;
 }
@@ -100,10 +107,10 @@ function normalizeDatasourceConfig(datasourceConfig) {
  * @param {Object|Object[]} datasourceConfigs - Datasource config(s)
  * @returns {{ systemConfig: Object, datasourceConfigs: Object[] }} Normalized configs
  */
-function normalizeWizardConfigs(systemConfig, datasourceConfigs) {
+function normalizeWizardConfigs(systemConfig, datasourceConfigs, openapiSpec) {
   const normalizedSystem = normalizeSystemConfig(systemConfig);
   const configs = Array.isArray(datasourceConfigs) ? datasourceConfigs : [datasourceConfigs];
-  const normalizedDatasources = configs.map(normalizeDatasourceConfig);
+  const normalizedDatasources = configs.map(config => normalizeDatasourceConfig(config, openapiSpec));
   return { systemConfig: normalizedSystem, datasourceConfigs: normalizedDatasources };
 }