@aifabrix/builder 2.49.2 → 2.51.0

package/lib/commands/up-builder-api-bootstrap.js

@@ -0,0 +1,71 @@
+/**
+ * Headless builder-api bootstrap provisioning (plan 154.2 / 197 pattern).
+ *
+ * @fileoverview up-builder-api bootstrap path when Docker + ENCRYPTION_KEY available
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const { callBootstrapApi } = require('../utils/app-bootstrap-api');
+const {
+  loadVariablesYaml,
+  extractAppConfiguration
+} = require('../utils/app-register-config');
+const { buildRegistrationData, saveLocalCredentials, applyLocalhostPortAdjustment } = require('../app/register');
+const { formatSuccessLine } = require('../utils/cli-test-layout-chalk');
+const { resolveBootstrapEncryptionKey } = require('./up-dataplane-bootstrap');
+
+/**
+ * Register builder-api via bootstrap endpoint (idempotent).
+ * @async
+ * @param {Object} options - up-builder-api options
+ * @param {string} controllerUrl
+ * @param {string} environmentKey
+ * @param {Function} buildBuilderApiImageRefFn
+ * @returns {Promise<void>}
+ */
+async function bootstrapBuilderApiApplication(options, controllerUrl, environmentKey, buildBuilderApiImageRefFn) {
+  const encryptionKey = await resolveBootstrapEncryptionKey();
+  if (!encryptionKey) {
+    throw new Error(
+      'Bootstrap encryption key is not available. Ensure miso-controller-secrets-encryptionKeyVault ' +
+        'exists in merged secrets after up-miso resolve.'
+    );
+  }
+
+  const { variables } = await loadVariablesYaml('builder-api');
+  const appConfig = await extractAppConfiguration(variables, 'builder-api', options);
+  await applyLocalhostPortAdjustment(appConfig, controllerUrl, options);
+
+  const imageOverride = options.image || buildBuilderApiImageRefFn(options);
+  const registerOpts = {
+    imageOverride,
+    image: imageOverride,
+    registryMode: options.registryMode,
+    envKey: environmentKey
+  };
+  const registrationData = buildRegistrationData(appConfig, registerOpts);
+
+  logger.log(chalk.blue('Provisioning builder-api via bootstrap (no device login)...'));
+  const responseData = await callBootstrapApi(
+    controllerUrl,
+    encryptionKey,
+    environmentKey,
+    registrationData
+  );
+
+  if (responseData.credentials && responseData.credentials.clientSecret) {
+    await saveLocalCredentials(responseData, controllerUrl);
+    logger.log(formatSuccessLine('Builder-api bootstrap credentials saved locally'));
+  } else if (responseData.status === 'already_provisioned') {
+    logger.log(formatSuccessLine('Builder-api already provisioned (bootstrap idempotent)'));
+  }
+}
+
+module.exports = {
+  bootstrapBuilderApiApplication
+};

package/lib/commands/up-builder-api-provision.js

@@ -0,0 +1,241 @@
+/**
+ * Register, deploy, and env refresh for builder-api (plan 154.2).
+ *
+ * @fileoverview Shared provision helpers for up-builder-api
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const chalk = require('chalk');
+const pathsUtil = require('../utils/paths');
+const { loadConfigFile } = require('../utils/config-format');
+const { resolveDockerImageRef, normalizeDockerRegistryPrefix } = require('../utils/resolve-docker-image-ref');
+const logger = require('../utils/logger');
+const config = require('../core/config');
+const { checkAuthentication } = require('../utils/app-register-auth');
+const { resolveControllerUrl } = require('../utils/controller-url');
+const { resolveEnvironment } = require('../core/config');
+const { registerApplication } = require('../app/register');
+const { rotateSecret } = require('../app/rotate-secret');
+const { checkApplicationExists } = require('../utils/app-existence');
+const { checkHealthEndpoint } = require('../utils/health-check');
+const app = require('../app');
+const { assertDevInfraUp } = require('./dev-infra-gate');
+const { publishedHostPort, parseDeveloperIdNum } = require('../utils/declarative-url-ports');
+const { canUseBootstrapProvision } = require('./up-dataplane-bootstrap');
+const { bootstrapBuilderApiApplication } = require('./up-builder-api-bootstrap');
+const { formatSuccessLine } = require('../utils/cli-test-layout-chalk');
+const { generateEnvFile } = require('../core/secrets');
+
+const CONTROLLER_HEALTH_PATH = '/health';
+const CONTROLLER_HEALTH_RETRY_ATTEMPTS = 30;
+const CONTROLLER_HEALTH_RETRY_DELAY_MS = 2000;
+
+/**
+ * @param {number} ms
+ * @returns {Promise<void>}
+ */
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * @param {string} baseUrl
+ * @returns {Promise<boolean>}
+ */
+async function isControllerHealthy(baseUrl) {
+  const healthUrl = `${baseUrl.replace(/\/+$/, '')}${CONTROLLER_HEALTH_PATH}`;
+  try {
+    return await checkHealthEndpoint(healthUrl, false);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * @param {string} baseUrl
+ * @param {{ attempts?: number, delayMs?: number }} [opts]
+ * @returns {Promise<boolean>}
+ */
+async function waitForControllerHealthy(baseUrl, opts = {}) {
+  const attempts = opts.attempts ?? CONTROLLER_HEALTH_RETRY_ATTEMPTS;
+  const delayMs = opts.delayMs ?? CONTROLLER_HEALTH_RETRY_DELAY_MS;
+  const normalized = baseUrl.replace(/\/+$/, '');
+  for (let i = 0; i < attempts; i += 1) {
+    if (await isControllerHealthy(normalized)) {
+      return true;
+    }
+    if (i < attempts - 1) {
+      await sleep(delayMs);
+    }
+  }
+  return false;
+}
+
+/**
+ * @param {Object} [options]
+ * @returns {string|undefined}
+ */
+function buildBuilderApiImageRef(options = {}) {
+  try {
+    const builderPath = pathsUtil.getBuilderPath('builder-api');
+    const configPath = pathsUtil.resolveApplicationConfigPath(builderPath);
+    const variables = loadConfigFile(configPath) || {};
+    const cli = normalizeDockerRegistryPrefix(options.registry);
+    const manifestReg = normalizeDockerRegistryPrefix(variables.image?.registry);
+    if (!cli && !manifestReg) {
+      return undefined;
+    }
+    const { imageName, imageTag } = resolveDockerImageRef('builder-api', variables, {
+      registry: options.registry || undefined
+    });
+    return `${imageName}:${imageTag}`;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * @param {Object} cfg
+ */
+function assertDevEnvironmentForBuilderApi(cfg) {
+  const environment = cfg && cfg.environment ? cfg.environment : 'dev';
+  if (environment !== 'dev') {
+    throw new Error(
+      'Builder-api is only supported in dev environment. Set with: aifabrix auth --set-environment dev.'
+    );
+  }
+}
+
+/**
+ * @async
+ * @param {Object} options
+ * @returns {Promise<string>}
+ */
+async function prepareHealthyControllerUrl(options) {
+  if (options.skipInfraCheck !== true) {
+    await assertDevInfraUp();
+  }
+  let controllerUrl = await resolveControllerUrl();
+  controllerUrl = controllerUrl.replace(/\/+$/, '');
+  const healthy = await waitForControllerHealthy(controllerUrl);
+  if (!healthy) {
+    throw new Error(`Controller at ${controllerUrl} is not responding. Ensure miso-controller is running.`);
+  }
+  return controllerUrl;
+}
+
+/**
+ * @async
+ * @param {Object} options
+ * @param {string} environmentKey
+ * @returns {Promise<{ controllerUrl: string, authConfig: Object|null, useBootstrap: boolean }>}
+ */
+async function resolveUpBuilderApiControllerContext(options, environmentKey) {
+  const useBootstrap = await canUseBootstrapProvision(options);
+  const controllerUrl = await prepareHealthyControllerUrl(options);
+  const cfg = await config.getConfig();
+  if (options.allowNonDev !== true) {
+    assertDevEnvironmentForBuilderApi(cfg);
+  }
+  if (useBootstrap) {
+    if (options.platformInstall !== true) {
+      logger.log(formatSuccessLine('Using bootstrap provision (no device login)'));
+    }
+    return { controllerUrl, authConfig: null, useBootstrap: true };
+  }
+  const authConfig = await checkAuthentication(controllerUrl, environmentKey, { throwOnFailure: true });
+  if (options.platformInstall !== true) {
+    logger.log(formatSuccessLine(`Logged in (environment: ${environmentKey})`));
+  }
+  return { controllerUrl, authConfig, useBootstrap: false };
+}
+
+/**
+ * @async
+ */
+async function registerOrRotateBuilderApi(options, controllerUrl, environmentKey, authConfig) {
+  const appExists = await checkApplicationExists('builder-api', controllerUrl, environmentKey, authConfig);
+  if (appExists) {
+    logger.log(chalk.blue('Builder-api already registered; rotating secret...'));
+    await rotateSecret('builder-api', options);
+    return;
+  }
+  const imageOverride = options.image || buildBuilderApiImageRef(options);
+  await registerApplication('builder-api', {
+    imageOverride,
+    image: imageOverride,
+    registryMode: options.registryMode,
+    envKey: environmentKey
+  });
+}
+
+/**
+ * @async
+ */
+async function deployBuilderApiToController(options) {
+  const imageOverride = options.image || buildBuilderApiImageRef(options);
+  await app.deployApp('builder-api', {
+    imageOverride,
+    image: imageOverride,
+    registryMode: options.registryMode,
+    envKey: options.envKey ? String(options.envKey).trim().toLowerCase() : undefined,
+    silentPoll: options.platformInstall === true
+  });
+}
+
+/**
+ * @async
+ */
+async function provisionAndDeployBuilderApi(options, environmentKey, controllerUrl, useBootstrap, authConfig) {
+  if (useBootstrap) {
+    await bootstrapBuilderApiApplication(options, controllerUrl, environmentKey, buildBuilderApiImageRef);
+  } else {
+    await registerOrRotateBuilderApi(options, controllerUrl, environmentKey, authConfig);
+  }
+
+  await deployBuilderApiToController({
+    ...options,
+    envKey: environmentKey,
+    ...(useBootstrap ? { deploymentAuth: 'client-credentials' } : {})
+  });
+  logger.log('');
+}
+
+/**
+ * Materialize builder-api .env after bootstrap/register so run uses fresh MISO_* credentials.
+ * @async
+ */
+async function refreshBuilderApiEnvAfterProvision() {
+  try {
+    await generateEnvFile('builder-api', null, 'local', true);
+    logger.log(formatSuccessLine('Refreshed builder-api .env from resolved secrets'));
+  } catch (error) {
+    logger.warn(chalk.yellow(`Could not refresh builder-api .env: ${error.message}`));
+  }
+}
+
+/**
+ * @async
+ */
+async function computeBuilderApiRunOverrides(options) {
+  const runEnvKey = options.runEnvKey ? String(options.runEnvKey).trim().toLowerCase() : undefined;
+  const developerIdRaw = await config.getDeveloperId();
+  const developerIdNum = parseDeveloperIdNum(developerIdRaw);
+  if (runEnvKey && runEnvKey !== 'dev' && runEnvKey !== 'tst') {
+    return { runEnvKey, portOverride: undefined };
+  }
+  const portOverride = publishedHostPort(3005, developerIdNum);
+  return { runEnvKey, portOverride };
+}
+
+module.exports = {
+  buildBuilderApiImageRef,
+  resolveUpBuilderApiControllerContext,
+  provisionAndDeployBuilderApi,
+  refreshBuilderApiEnvAfterProvision,
+  computeBuilderApiRunOverrides,
+  resolveEnvironment
+};

package/lib/commands/up-builder-api.js

@@ -0,0 +1,88 @@
+const { formatSuccessParagraph } = require('../utils/cli-test-layout-chalk');
+/**
+ * AI Fabrix Builder - Up Builder API Command
+ *
+ * Materializes builder-api from template, registers via bootstrap or device login,
+ * deploys manifest to controller, and runs locally (plan 154.2).
+ *
+ * @fileoverview up-builder-api command implementation
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const app = require('../app');
+const {
+  ensureBuilderApiAppsFromTemplate,
+  patchEnvOutputPathForDeployOnly,
+  validateEnvOutputPathFolderOrNull,
+  refreshUrlsLocalRegistryForCurrentProject
+} = require('./up-common');
+const {
+  resolveUpBuilderApiControllerContext,
+  provisionAndDeployBuilderApi,
+  refreshBuilderApiEnvAfterProvision,
+  computeBuilderApiRunOverrides,
+  resolveEnvironment
+} = require('./up-builder-api-provision');
+
+/**
+ * @async
+ * @param {Object} [options]
+ * @returns {Promise<void>}
+ */
+async function handleUpBuilderApi(options = {}) {
+  if (options.platformInstall !== true) {
+    logger.log(
+      chalk.blue('Starting up-builder-api (bootstrap/register, deploy, then run builder-api locally)...\n')
+    );
+  }
+
+  const environmentKey =
+    options && options.envKey
+      ? String(options.envKey).trim().toLowerCase()
+      : await resolveEnvironment();
+
+  const { controllerUrl, authConfig, useBootstrap } = await resolveUpBuilderApiControllerContext(
+    options,
+    environmentKey
+  );
+
+  await ensureBuilderApiAppsFromTemplate({ silent: options.platformInstall === true });
+  validateEnvOutputPathFolderOrNull('builder-api');
+  patchEnvOutputPathForDeployOnly('builder-api');
+  refreshUrlsLocalRegistryForCurrentProject();
+
+  const { emitSystemBuilderAppManifestLineIfTTY } = require('../utils/manifest-source-emit');
+  emitSystemBuilderAppManifestLineIfTTY(logger, 'builder-api');
+
+  await provisionAndDeployBuilderApi(options, environmentKey, controllerUrl, useBootstrap, authConfig);
+  await refreshBuilderApiEnvAfterProvision();
+
+  const { portOverride } = await computeBuilderApiRunOverrides(options);
+
+  const runOpts = {
+    registry: options.registry,
+    registryMode: options.registryMode,
+    skipEnvOutputPath: true,
+    skipInfraCheck: true,
+    skipManifestMetadataLine: true,
+    ...(portOverride !== undefined ? { port: portOverride } : {}),
+    ...(options.platformInstall === true ? { silentPoll: true } : {})
+  };
+  if (options.image) {
+    runOpts.image = options.image;
+  }
+  await app.runApp('builder-api', runOpts);
+
+  if (options.platformInstall !== true) {
+    logger.log(
+      formatSuccessParagraph(
+        'up-builder-api complete. Builder-api is registered, deployed in dev, and running locally.'
+      )
+    );
+  }
+}
+
+module.exports = { handleUpBuilderApi };

package/lib/commands/up-common.js

@@ -328,11 +328,15 @@ async function ensureAppFromTemplate(appName, opts = {}) {
 /** Dataplane app key materialized for dataplane install (worker/beat are compose sidecars). */
 const DATAPLANE_BUILDER_APP_KEYS = Object.freeze(['dataplane']);
 
+/** Builder API platform app key (154.2). */
+const BUILDER_API_APP_KEYS = Object.freeze(['builder-api']);
+
 /** Builder dirs removed on up-platform / setup force (includes Celery companions). */
 const PLATFORM_BUILDER_CLEAN_APP_KEYS = Object.freeze([
   'keycloak',
   'miso-controller',
-  ...DATAPLANE_BUILDER_APP_KEYS
+  ...DATAPLANE_BUILDER_APP_KEYS,
+  ...BUILDER_API_APP_KEYS
 ]);
 
 /**
@@ -354,6 +358,23 @@ async function ensureDataplaneAppsFromTemplate(opts = {}) {
   return anyCopied;
 }
 
+/**
+ * Ensure builder-api manifest exists under the system builder root.
+ *
+ * @param {object} [opts]
+ * @returns {Promise<boolean>}
+ */
+async function ensureBuilderApiAppsFromTemplate(opts = {}) {
+  let anyCopied = false;
+  for (const appName of BUILDER_API_APP_KEYS) {
+    const copied = await ensureAppFromTemplate(appName, opts);
+    if (copied) {
+      anyCopied = true;
+    }
+  }
+  return anyCopied;
+}
+
 /**
  * Merge builder/packages `application.yaml` into `~/.aifabrix/urls.local.yaml`.
  * Same scan as the end of {@link prepareUrlsLocalRegistryForUpPlatform}; call after platform app
@@ -381,15 +402,18 @@ async function prepareUrlsLocalRegistryForUpPlatform(opts = {}) {
   await ensureAppFromTemplate('keycloak', templateOpts);
   await ensureAppFromTemplate('miso-controller', templateOpts);
   await ensureDataplaneAppsFromTemplate(templateOpts);
+  await ensureBuilderApiAppsFromTemplate(templateOpts);
   refreshUrlsLocalRegistryForCurrentProject();
 }
 
 module.exports = {
   applyUpPlatformForceConfig,
   cleanBuilderAppDirs,
+  BUILDER_API_APP_KEYS,
   DATAPLANE_BUILDER_APP_KEYS,
   PLATFORM_BUILDER_CLEAN_APP_KEYS,
   ensureAppFromTemplate,
+  ensureBuilderApiAppsFromTemplate,
   ensureDataplaneAppsFromTemplate,
   patchEnvOutputPathForDeployOnly,
   validateEnvOutputPathFolderOrNull,

package/lib/commands/upload-credential-push-log.js

@@ -0,0 +1,117 @@
+/**
+ * Credential + runtime config push with compact/verbose logging for upload publish.
+ *
+ * @fileoverview Upload credential push helpers
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const path = require('path');
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const { formatSuccessLine } = require('../utils/cli-test-layout-chalk');
+const { resolveExternalSystemPath } = require('../utils/app-path-resolve');
+const { pushCredentialSecrets } = require('../utils/credential-secrets-env');
+const { runWithCredentialPushSpinner } = require('../utils/credential-push-ui');
+const { collectResolvedVariableConfigurationNames } = require('../utils/configuration-env-resolver');
+const { pushRuntimeConfigurationValues } = require('../utils/runtime-configuration-values');
+
+/**
+ * @param {Object} pushResult
+ * @param {string[]} resolvedParamNames
+ * @param {boolean} quietLog
+ */
+function logCredentialPushOutcome(pushResult, resolvedParamNames, quietLog) {
+  if (quietLog) {
+    return;
+  }
+  if (pushResult.pushed > 0) {
+    const keyList = pushResult.keys?.length ? ` (${pushResult.keys.join(', ')})` : '';
+    logger.log(
+      formatSuccessLine(`Pushed ${pushResult.pushed} credential secret(s) to dataplane${keyList}.`)
+    );
+  } else {
+    logger.log(chalk.yellow('Secret push skipped'));
+  }
+  if (resolvedParamNames.length > 0) {
+    const nameList = ` (${resolvedParamNames.join(', ')})`;
+    logger.log(
+      formatSuccessLine(
+        `Resolved ${resolvedParamNames.length} integration parameter(s) from .env for publish${nameList}.`
+      )
+    );
+  }
+  if (pushResult.warning) {
+    logger.log(chalk.yellow(`Warning: ${pushResult.warning}`));
+  }
+}
+
+/**
+ * @async
+ * @returns {Promise<{ pushed: number, names: string[] }>}
+ */
+async function pushRuntimeConfigWithLog(dataplaneUrl, authConfig, systemKey, payload, quietLog) {
+  try {
+    const configPush = await pushRuntimeConfigurationValues(dataplaneUrl, authConfig, systemKey, payload);
+    if (configPush.pushed > 0 && !quietLog) {
+      const nameList = configPush.names.length ? ` (${configPush.names.join(', ')})` : '';
+      logger.log(
+        formatSuccessLine(
+          `Stored ${configPush.pushed} runtime configuration value(s) in dataplane${nameList}.`
+        )
+      );
+    }
+    return configPush;
+  } catch (err) {
+    if (!quietLog) {
+      logger.log(chalk.yellow(`Warning: runtime configuration store skipped: ${err.message}`));
+    }
+    return { pushed: 0, names: [] };
+  }
+}
+
+/**
+ * Pushes credential secrets from .env and payload to dataplane; logs result or warning.
+ *
+ * @async
+ * @param {string} dataplaneUrl
+ * @param {Object} authConfig
+ * @param {string} systemKey
+ * @param {Object} payload
+ * @param {Object} [uiOpts]
+ * @returns {Promise<Object>}
+ */
+async function pushAndLogCredentialSecrets(dataplaneUrl, authConfig, systemKey, payload, uiOpts = {}) {
+  const pathOptions = uiOpts.pathOptions || {};
+  const envFilePath = path.join(resolveExternalSystemPath(systemKey, pathOptions), '.env');
+  const pushResult = await runWithCredentialPushSpinner(
+    () =>
+      pushCredentialSecrets(dataplaneUrl, authConfig, {
+        envFilePath,
+        appName: systemKey,
+        payload
+      }),
+    { systemKey, json: uiOpts.json === true }
+  );
+  const resolvedParamNames = collectResolvedVariableConfigurationNames(payload);
+  const quietLog = uiOpts.suppressCredentialLog === true;
+  logCredentialPushOutcome(pushResult, resolvedParamNames, quietLog);
+  const configPush = await pushRuntimeConfigWithLog(
+    dataplaneUrl,
+    authConfig,
+    systemKey,
+    payload,
+    quietLog
+  );
+  return {
+    credentialPush: { pushed: pushResult.pushed || 0, keys: pushResult.keys || [] },
+    configPush,
+    resolvedParamNames
+  };
+}
+
+module.exports = {
+  pushAndLogCredentialSecrets
+};

package/lib/commands/upload-data-sync-finalize.js

@@ -0,0 +1,60 @@
+/**
+ * @fileoverview Finalize upload with optional datasource data sync.
+ * @author AI Fabrix Team
+ * @version 1.0.0
+ */
+
+'use strict';
+
+const { runPostPublishDataSync } = require('./run-post-publish-data-sync');
+const { runUploadSidecarFollowups } = require('./upload-sidecar-followups');
+const { formatSuccessLine } = require('../utils/cli-test-layout-chalk');
+const logger = require('../utils/logger');
+const { isQuietMinimalSync } = require('../utils/upload-sync-options');
+
+/**
+ * @async
+ * @param {Object} ctx
+ * @param {Object} payload
+ * @param {Object} options
+ * @returns {Promise<Object>}
+ */
+async function finalizeUploadPublishResult(ctx, payload, options) {
+  const dataSync = await runPostPublishDataSync({
+    dataplaneUrl: ctx.dataplaneUrl,
+    authConfig: ctx.authConfig,
+    payload,
+    options,
+    label: 'Upload data sync'
+  });
+  return { ...ctx, dataSync };
+}
+
+/**
+ * @async
+ * @param {Object} params
+ * @returns {Promise<Object>}
+ */
+async function finishUnchangedUploadPublication(params) {
+  const { systemKey, options, payload, dataplaneUrl, authConfig, environment } = params;
+  if (!isQuietMinimalSync(options)) {
+    logger.log(formatSuccessLine('Config unchanged on dataplane — upload skipped.'));
+  }
+  await runUploadSidecarFollowups({
+    systemKey,
+    options,
+    payload,
+    dataplaneUrl,
+    authConfig
+  });
+  return finalizeUploadPublishResult(
+    { dataplaneUrl, authConfig, environment, uploadSkipped: true },
+    payload,
+    options
+  );
+}
+
+module.exports = {
+  finalizeUploadPublishResult,
+  finishUnchangedUploadPublication
+};