@aifabrix/builder 2.42.1 → 2.44.0

package/lib/datasource/deploy.js

@@ -17,6 +17,16 @@ const { publishDatasourceViaPipeline } = require('../api/pipeline.api');
 const { formatApiError } = require('../utils/api-error-handler');
 const logger = require('../utils/logger');
 const { logDataplanePipelineWarning } = require('../utils/dataplane-pipeline-warning');
+const {
+  sectionTitle,
+  headerKeyValue,
+  metadata,
+  infoLine,
+  formatStatusKeyValue,
+  formatBlockingError,
+  successGlyph,
+  failureGlyph
+} = require('../utils/cli-test-layout-chalk');
 const {
   buildResolvedEnvMapForIntegration,
   resolveConfigurationValues
@@ -64,41 +74,41 @@ async function getDataplaneUrl(controllerUrl, appKey, environment, authConfig) {
 }
 
 /**
- * Validate deployment inputs
- * @param {string} appKey - Application key
- * @param {string} filePath - File path
- * @param {Object} options - Options
+ * Validate deploy CLI input (file path or datasource key, same rules as `datasource validate`).
+ * @param {string} fileOrKey - Path to JSON or datasource `key` under integration/<app>/
  * @throws {Error} If validation fails
  */
-function validateDeploymentInputs(appKey, filePath) {
-  if (!appKey || typeof appKey !== 'string') {
-    throw new Error('Application key is required');
-  }
-  if (!filePath || typeof filePath !== 'string') {
-    throw new Error('File path is required');
+function validateDeployFileOrKeyInput(fileOrKey) {
+  if (!fileOrKey || typeof fileOrKey !== 'string' || !fileOrKey.trim()) {
+    throw new Error('File path or datasource key is required');
   }
 }
 
 /**
- * Validate and load datasource file
+ * Validate and load datasource file (path or datasource key, resolved like `datasource validate`).
  * @async
- * @param {string} filePath - Path to datasource file
+ * @param {string} filePathOrKey - Path to datasource JSON or datasource `key`
  * @returns {Promise<Object>} Datasource configuration
  * @throws {Error} If validation or loading fails
  */
-async function validateAndLoadDatasourceFile(filePath) {
-  logger.log(chalk.blue('🔍 Validating datasource file...'));
-  const validation = await validateDatasourceFile(filePath);
+async function validateAndLoadDatasourceFile(filePathOrKey) {
+  logger.log(infoLine('ℹ Validating datasource file'));
+  const validation = await validateDatasourceFile(filePathOrKey);
   if (!validation.valid) {
-    logger.error(chalk.red('❌ Datasource validation failed:'));
+    logger.log('');
+    logger.error(formatBlockingError('Datasource validation failed'));
     validation.errors.forEach(error => {
-      logger.error(chalk.red(`  • ${error}`));
+      logger.error(formatBlockingError(error));
     });
     throw new Error('Datasource file validation failed');
   }
-  logger.log(chalk.green('✓ Datasource file is valid'));
 
-  const content = fs.readFileSync(filePath, 'utf8');
+  const resolvedPath = validation.resolvedPath;
+  logger.log(headerKeyValue('File:', resolvedPath));
+  logger.log(`${successGlyph()} ${chalk.white('Datasource file is valid.')}`);
+  logger.log('');
+
+  const content = fs.readFileSync(resolvedPath, 'utf8');
   try {
     return JSON.parse(content);
   } catch (error) {
@@ -116,28 +126,28 @@ async function validateAndLoadDatasourceFile(filePath) {
  */
 async function setupDeploymentAuth(controllerUrl, environment, appKey) {
   const { resolveDataplaneUrl } = require('../utils/dataplane-resolver');
-  logger.log(chalk.blue('🔐 Getting authentication...'));
+  logger.log(infoLine('ℹ Resolving authentication'));
   const authConfig = await getDeploymentAuth(controllerUrl, environment, appKey);
-  logger.log(chalk.green('✓ Authentication successful'));
+  logger.log(`${successGlyph()} ${chalk.white('Authentication ready')}`);
 
-  logger.log(chalk.blue('🌐 Resolving dataplane URL...'));
+  logger.log(infoLine('ℹ Resolving dataplane URL'));
   let dataplaneUrl;
   try {
     dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig);
-    logger.log(chalk.green(`✓ Dataplane URL: ${dataplaneUrl}`));
+    logger.log(`${metadata('Dataplane:')} ${chalk.cyan(dataplaneUrl)}`);
   } catch (error) {
-    logger.error(chalk.red('❌ Failed to resolve dataplane URL:'), error.message);
-    logger.error(chalk.gray('\nThe dataplane URL is automatically discovered from the controller.'));
-    logger.error(chalk.gray('If discovery fails, ensure you are logged in and the controller is accessible:'));
-    logger.error(chalk.gray('   aifabrix login'));
+    logger.error(`${failureGlyph()} ${chalk.red('Failed to resolve dataplane URL:')} ${chalk.red(error.message)}`);
+    logger.error(metadata('The dataplane URL is automatically discovered from the controller.'));
+    logger.error(metadata('If discovery fails, ensure you are logged in and the controller is accessible:'));
+    logger.error(metadata('aifabrix login'));
     throw error;
   }
 
   // Validate dataplane URL
   if (!dataplaneUrl || !dataplaneUrl.trim()) {
-    logger.error(chalk.red('❌ Dataplane URL is empty.'));
-    logger.error(chalk.gray('The dataplane URL could not be discovered from the controller.'));
-    logger.error(chalk.gray('Ensure the dataplane service is registered in the controller.'));
+    logger.error(`${failureGlyph()} ${chalk.red('Dataplane URL is empty.')}`);
+    logger.error(metadata('The dataplane URL could not be discovered from the controller.'));
+    logger.error(metadata('Ensure the dataplane service is registered in the controller.'));
     throw new Error('Dataplane URL is empty');
   }
 
@@ -156,30 +166,32 @@ async function setupDeploymentAuth(controllerUrl, environment, appKey) {
  */
 async function publishDatasourceToDataplane(dataplaneUrl, systemKey, authConfig, datasourceConfig) {
   requireBearerForDataplanePipeline(authConfig);
-  logger.log(chalk.blue('\n🚀 Publishing datasource to dataplane...'));
+  logger.log('');
+  logger.log(sectionTitle('Publish'));
   logDataplanePipelineWarning();
 
   const publishResponse = await publishDatasourceViaPipeline(dataplaneUrl, systemKey, authConfig, datasourceConfig);
 
   if (!publishResponse.success) {
     const formattedError = publishResponse.formattedError || formatApiError(publishResponse);
-    logger.error(chalk.red('❌ Publish failed:'));
-    logger.error(formattedError);
+    logger.error(`${failureGlyph()} ${chalk.red('Publish failed:')} ${chalk.red(formattedError)}`);
 
     // Show dataplane URL and endpoint information
     if (publishResponse.errorData && publishResponse.errorData.endpointUrl) {
-      logger.error(chalk.gray(`\nEndpoint URL: ${publishResponse.errorData.endpointUrl}`));
+      logger.error(
+        `\n${metadata('Endpoint URL:')} ${chalk.cyan(publishResponse.errorData.endpointUrl)}`
+      );
     } else if (dataplaneUrl) {
-      logger.error(chalk.gray(`\nDataplane URL: ${dataplaneUrl}`));
-      logger.error(chalk.gray(`System Key: ${systemKey}`));
+      logger.error(`\n${metadata('Dataplane URL:')} ${chalk.cyan(dataplaneUrl)}`);
+      logger.error(headerKeyValue('System Key:', systemKey));
     }
 
-    logger.error(chalk.gray('\nFull response for debugging:'));
-    logger.error(chalk.gray(JSON.stringify(publishResponse, null, 2)));
+    logger.error(metadata('\nFull response for debugging:'));
+    logger.error(metadata(JSON.stringify(publishResponse, null, 2)));
     throw new Error(`Dataplane publish failed: ${formattedError}`);
   }
 
-  logger.log(chalk.green('\n✓ Datasource published successfully!'));
+  logger.log(`${successGlyph()} ${chalk.white('Configuration published to dataplane.')}`);
   return publishResponse;
 }
 
@@ -190,9 +202,21 @@ async function publishDatasourceToDataplane(dataplaneUrl, systemKey, authConfig,
  * @param {string} environment - Environment key
  */
 function displayDeploymentResults(datasourceConfig, systemKey, environment) {
-  logger.log(chalk.blue(`\nDatasource: ${datasourceConfig.key || datasourceConfig.displayName}`));
-  logger.log(chalk.blue(`System: ${systemKey}`));
-  logger.log(chalk.blue(`Environment: ${environment}`));
+  const datasourceLabel = datasourceConfig.key || datasourceConfig.displayName || '(unknown)';
+  logger.log('');
+  logger.log(sectionTitle('Result'));
+  logger.log(headerKeyValue('Datasource:', datasourceLabel));
+  logger.log(headerKeyValue('System:', systemKey));
+  logger.log(headerKeyValue('Environment:', environment));
+  logger.log('');
+  logger.log(formatStatusKeyValue('ok', '✔'));
+}
+
+function logDatasourceUploadSectionHeader() {
+  logger.log('');
+  logger.log(sectionTitle('Datasource upload'));
+  logger.log(metadata('Publish one datasource JSON to the dataplane'));
+  logger.log('');
 }
 
 /**
@@ -201,18 +225,17 @@ function displayDeploymentResults(datasourceConfig, systemKey, environment) {
  *
  * @async
  * @function deployDatasource
- * @param {string} appKey - Application key
- * @param {string} filePath - Path to datasource JSON file
+ * @param {string} fileOrKey - Path to datasource JSON file, or datasource `key` under integration/<app>/ (same resolution as `datasource validate`)
  * @param {Object} [_options] - Deployment options (reserved)
  * @returns {Promise<Object>} Deployment result
  * @throws {Error} If deployment fails
  */
-async function deployDatasource(appKey, filePath, _options) {
+async function deployDatasource(fileOrKey, _options) {
   const { resolveControllerUrl } = require('../utils/controller-url');
   const { resolveEnvironment } = require('../core/config');
   const { displayCommandHeader } = require('../utils/command-header');
 
-  validateDeploymentInputs(appKey, filePath);
+  validateDeployFileOrKeyInput(fileOrKey);
 
   // Resolve controller and environment from config
   const controllerUrl = await resolveControllerUrl();
@@ -220,13 +243,12 @@ async function deployDatasource(appKey, filePath, _options) {
 
   // Display command header
   displayCommandHeader(controllerUrl, environment);
+  logDatasourceUploadSectionHeader();
 
-  logger.log(chalk.blue('📋 Deploying datasource...\n'));
-
-  // Validate and load datasource file
-  const datasourceConfig = await validateAndLoadDatasourceFile(filePath);
+  // Validate and load datasource file (resolves key → path like validate)
+  const datasourceConfig = await validateAndLoadDatasourceFile(fileOrKey.trim());
 
-  // Extract systemKey
+  // Extract systemKey (required in JSON; also used as controller app key for external systems)
   const systemKey = datasourceConfig.systemKey;
   if (!systemKey) {
     throw new Error('systemKey is required in datasource configuration');
@@ -237,8 +259,8 @@ async function deployDatasource(appKey, filePath, _options) {
     resolveConfigurationValues(datasourceConfig.configuration, envMap, secrets, systemKey);
   }
 
-  // Setup authentication and get dataplane URL
-  const { authConfig, dataplaneUrl } = await setupDeploymentAuth(controllerUrl, environment, appKey);
+  // Setup authentication and get dataplane URL (application key matches systemKey for external integrations)
+  const { authConfig, dataplaneUrl } = await setupDeploymentAuth(controllerUrl, environment, systemKey);
 
   // Publish to dataplane
   await publishDatasourceToDataplane(dataplaneUrl, systemKey, authConfig, datasourceConfig);

package/lib/datasource/field-reference-validator.js

@@ -11,78 +11,119 @@
  */
 
 /**
- * Validates that all field references in indexing, validation, and quality
- * exist in fieldMappings.attributes. When fieldMappings.attributes is missing
- * or empty, returns no errors (skip check, matching dataplane behavior).
+ * Set of attribute names plus root dimension keys valid for field references.
+ * Used for primaryKey (schema allows dimension keys or attributes) and other paths (attributes only).
  *
- * @function validateFieldReferences
- * @param {Object} parsed - Parsed datasource object (after JSON parse)
- * @returns {string[]} Array of error messages; empty if no invalid references
- *
- * @example
- * const errors = validateFieldReferences(parsed);
- * if (errors.length > 0) {
- *   errors.forEach(e => console.error(e));
- * }
+ * @param {Object} parsed - Parsed datasource object
+ * @returns {{ attributes: string[], attributesAndDimensions: Set<string> }}
  */
-function validateFieldReferences(parsed) {
-  const errors = [];
-  const normalizedAttributes = Object.keys(
-    parsed?.fieldMappings?.attributes ?? {}
-  );
+function getNormalizedSets(parsed) {
+  const attributes = Object.keys(parsed?.fieldMappings?.attributes ?? {});
+  const rootDims = Object.keys(parsed?.dimensions ?? {}).filter(k => {
+    const b = parsed.dimensions[k];
+    return b && typeof b === 'object';
+  });
+  const attributesAndDimensions = new Set([...attributes, ...rootDims]);
+  return { attributes, attributesAndDimensions };
+}
 
-  if (normalizedAttributes.length === 0) {
-    return [];
-  }
+/** @param {string[]} errors */
+function checkPrimaryKey(parsed, attributesAndDimensions, errors) {
+  const primaryKey = parsed?.primaryKey;
+  if (!Array.isArray(primaryKey)) return;
+  primaryKey.forEach((field, i) => {
+    if (typeof field === 'string' && field !== '' && !attributesAndDimensions.has(field)) {
+      errors.push(
+        `primaryKey[${i}]: field '${field}' does not exist in fieldMappings.attributes or root dimensions. Each primaryKey value must reference an attribute or dimension key.`
+      );
+    }
+  });
+}
+
+/** @param {string[]} errors */
+function checkExposedProfiles(parsed, attrSet, errors) {
+  const profiles = parsed?.exposed?.profiles;
+  if (!profiles || typeof profiles !== 'object' || Array.isArray(profiles)) return;
+  Object.entries(profiles).forEach(([profileName, fields]) => {
+    if (!Array.isArray(fields)) return;
+    fields.forEach((field, idx) => {
+      if (typeof field === 'string' && !attrSet.has(field)) {
+        errors.push(
+          `exposed.profiles.${profileName}[${idx}]: field '${field}' does not exist in fieldMappings.attributes. Add the attribute or remove the reference.`
+        );
+      }
+    });
+  });
+}
 
-  // indexing.embedding: array of field names
+/** @param {string[]} errors */
+function checkIndexingAndValidation(parsed, normalizedAttributes, errors) {
   const embedding = parsed?.indexing?.embedding;
   if (Array.isArray(embedding)) {
     embedding.forEach((field, i) => {
       if (typeof field === 'string' && !normalizedAttributes.includes(field)) {
         errors.push(
-          `indexing.embedding[${i}]: field '${field}' does not exist in fieldMappings.attributes`
+          `indexing.embedding[${i}]: field '${field}' does not exist in fieldMappings.attributes. Add the attribute or remove the reference.`
         );
       }
     });
   }
-
-  // indexing.uniqueKey: single field name
   const uniqueKey = parsed?.indexing?.uniqueKey;
-  if (typeof uniqueKey === 'string' && uniqueKey !== '') {
-    if (!normalizedAttributes.includes(uniqueKey)) {
-      errors.push(
-        `indexing.uniqueKey: field '${uniqueKey}' does not exist in fieldMappings.attributes`
-      );
-    }
+  if (typeof uniqueKey === 'string' && uniqueKey !== '' && !normalizedAttributes.includes(uniqueKey)) {
+    errors.push(
+      `indexing.uniqueKey: field '${uniqueKey}' does not exist in fieldMappings.attributes. Add the attribute or remove the reference.`
+    );
   }
-
-  // validation.repeatingValues[].field
   const repeatingValues = parsed?.validation?.repeatingValues;
   if (Array.isArray(repeatingValues)) {
     repeatingValues.forEach((rule, index) => {
       const field = rule?.field;
       if (typeof field === 'string' && !normalizedAttributes.includes(field)) {
         errors.push(
-          `validation.repeatingValues[${index}].field: field '${field}' does not exist in fieldMappings.attributes`
+          `validation.repeatingValues[${index}].field: field '${field}' does not exist in fieldMappings.attributes. Add the attribute or remove the reference.`
         );
       }
     });
   }
-
-  // quality.rejectIf[].field
   const rejectIf = parsed?.quality?.rejectIf;
   if (Array.isArray(rejectIf)) {
     rejectIf.forEach((rule, index) => {
       const field = rule?.field;
       if (typeof field === 'string' && !normalizedAttributes.includes(field)) {
         errors.push(
-          `quality.rejectIf[${index}].field: field '${field}' does not exist in fieldMappings.attributes`
+          `quality.rejectIf[${index}].field: field '${field}' does not exist in fieldMappings.attributes. Add the attribute or remove the reference.`
         );
       }
     });
   }
+}
+
+/**
+ * Validates that all field references in indexing, validation, quality,
+ * primaryKey, and exposed.profiles exist in fieldMappings.attributes (or
+ * root dimension keys for primaryKey per schema). When fieldMappings.attributes is
+ * missing or empty, returns no errors (skip check, matching dataplane behavior).
+ *
+ * @function validateFieldReferences
+ * @param {Object} parsed - Parsed datasource object (after JSON parse)
+ * @returns {string[]} Array of error messages; empty if no invalid references
+ *
+ * @example
+ * const errors = validateFieldReferences(parsed);
+ * if (errors.length > 0) {
+ *   errors.forEach(e => console.error(e));
+ * }
+ */
+function validateFieldReferences(parsed) {
+  const errors = [];
+  const { attributes: normalizedAttributes, attributesAndDimensions } = getNormalizedSets(parsed);
+  const attrSet = new Set(normalizedAttributes);
+
+  checkPrimaryKey(parsed, attributesAndDimensions, errors);
+  checkExposedProfiles(parsed, attrSet, errors);
+  if (normalizedAttributes.length === 0) return errors;
 
+  checkIndexingAndValidation(parsed, normalizedAttributes, errors);
   return errors;
 }
 

package/lib/datasource/integration-context.js

@@ -0,0 +1,63 @@
+/**
+ * @fileoverview Shared integration app resolution (system key, datasource file lookup).
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const path = require('path');
+const fs = require('fs').promises;
+const { getIntegrationPath } = require('../utils/paths');
+const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
+const { loadConfigFile } = require('../utils/config-format');
+
+/**
+ * @param {string} appKey - Integration app key
+ * @returns {Promise<string>} systemKey
+ */
+async function getSystemKeyFromAppKey(appKey) {
+  const appPath = getIntegrationPath(appKey);
+  const configPath = resolveApplicationConfigPath(appPath);
+  const config = loadConfigFile(configPath);
+  if (!config.externalIntegration || !config.externalIntegration.systems || config.externalIntegration.systems.length === 0) {
+    throw new Error(`No externalIntegration.systems found in ${configPath}`);
+  }
+  const systemFile = config.externalIntegration.systems[0];
+  const systemPath = path.isAbsolute(systemFile)
+    ? systemFile
+    : path.join(appPath, systemFile);
+  const systemContent = await fs.readFile(systemPath, 'utf8');
+  const yaml = require('js-yaml');
+  const systemConfig = yaml.load(systemContent);
+  return systemConfig?.key || path.basename(systemFile, '-system.yaml').replace('-system', '');
+}
+
+/**
+ * Find a datasource filename by matching the key inside the file.
+ * @param {string} appPath
+ * @param {string} schemaBasePath
+ * @param {string[]} datasourceFiles
+ * @param {string} datasourceKey
+ * @returns {string|null}
+ */
+function findDatasourceFileByKey(appPath, schemaBasePath, datasourceFiles, datasourceKey) {
+  const fsSync = require('fs');
+  for (const f of datasourceFiles) {
+    if (!f || typeof f !== 'string') continue;
+    const fullPath = path.isAbsolute(schemaBasePath)
+      ? path.join(schemaBasePath, f)
+      : path.join(appPath, schemaBasePath, f);
+    if (!fsSync.existsSync(fullPath)) continue;
+    try {
+      const parsed = loadConfigFile(fullPath);
+      if (parsed && parsed.key === datasourceKey) return f;
+    } catch {
+      // skip unreadable or invalid files
+    }
+  }
+  return null;
+}
+
+module.exports = {
+  getSystemKeyFromAppKey,
+  findDatasourceFileByKey
+};

package/lib/datasource/list.js

@@ -1,7 +1,8 @@
+const { formatBlockingError } = require('../utils/cli-test-layout-chalk');
 /**
  * Datasource List Command
  *
- * Lists datasources from the dataplane (GET /api/v1/external/).
+ * Lists datasources from the dataplane (GET /api/v1/external).
  * Resolves dataplane URL from the controller, then calls the dataplane list API.
  *
  * @fileoverview Datasource listing for AI Fabrix Builder
@@ -74,7 +75,7 @@ function extractFromPaginatedFormat(apiResponse) {
  * @param {Object} apiResponse - API response object
  */
 function logInvalidResponseError(apiResponse) {
-  logger.error(chalk.red('❌ Invalid response: expected data array or items array'));
+  logger.error(formatBlockingError('Invalid response: expected data array or items array'));
   logger.error(chalk.gray('\nAPI response type:'), typeof apiResponse);
   logger.error(chalk.gray('API response:'), JSON.stringify(apiResponse, null, 2));
 }
@@ -177,7 +178,7 @@ async function getDeviceTokenFromConfig(config) {
  */
 function validateDatasourceListingAuth(token, controllerUrl) {
   if (!token || !controllerUrl || (typeof controllerUrl === 'string' && !controllerUrl.trim())) {
-    logger.error(chalk.red('❌ Not logged in. Run: aifabrix login'));
+    logger.error(formatBlockingError('Not logged in. Run: aifabrix login'));
     logger.error(chalk.gray('   Use device code flow: aifabrix login --method device --controller <url>'));
     process.exit(1);
   }
@@ -215,7 +216,7 @@ function handleDatasourceApiError(response, dataplaneUrl = null) {
 function validateControllerUrl(controllerUrl) {
   const trimmed = controllerUrl.trim();
   if (!trimmed) {
-    logger.error(chalk.red('❌ Controller URL is empty.'));
+    logger.error(formatBlockingError('Controller URL is empty.'));
     logger.error(chalk.gray(`   Controller URL from config: ${JSON.stringify(controllerUrl)}`));
     logger.error(chalk.gray('   Run: aifabrix login --method device --controller <url>'));
     process.exit(1);
@@ -238,7 +239,7 @@ async function resolveAndValidateDataplaneUrl(controllerUrl, environment, authCo
     // discoverDataplaneUrl already logs progress and success messages
     dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig);
   } catch (error) {
-    logger.error(chalk.red('❌ Failed to resolve dataplane URL:'), error.message);
+    logger.error(formatBlockingError('Failed to resolve dataplane URL:'), error.message);
     logger.error(chalk.gray('\nThe dataplane URL is automatically discovered from the controller.'));
     logger.error(chalk.gray('If discovery fails, ensure you are logged in and the controller is accessible:'));
     logger.error(chalk.gray('   aifabrix login'));
@@ -248,7 +249,7 @@ async function resolveAndValidateDataplaneUrl(controllerUrl, environment, authCo
   }
 
   if (!dataplaneUrl || typeof dataplaneUrl !== 'string' || !dataplaneUrl.trim()) {
-    logger.error(chalk.red('❌ Dataplane URL is empty.'));
+    logger.error(formatBlockingError('Dataplane URL is empty.'));
     logger.error(chalk.gray('The dataplane URL could not be discovered from the controller.'));
     logger.error(chalk.gray('Ensure the dataplane service is registered in the controller.'));
     process.exit(1);
@@ -295,7 +296,7 @@ async function listDatasources(_options) {
   // Resolve dataplane URL first (required for list call)
   const dataplaneUrl = await resolveAndValidateDataplaneUrl(controllerUrl, environment, authConfig);
 
-  // List datasources from dataplane (GET /api/v1/external/)
+  // List datasources from dataplane (GET /api/v1/external)
   const response = await listDatasourcesFromDataplane(dataplaneUrl, authConfig);
 
   if (!response.success || !response.data) {