@aifabrix/builder 2.43.0 → 2.44.0

package/lib/datasource/field-reference-validator.js

@@ -11,16 +11,19 @@
  */
 
 /**
- * Set of attribute names plus dimension keys valid for field references.
- * Used for primaryKey (schema allows dimensions or attributes) and other paths (attributes only).
+ * Set of attribute names plus root dimension keys valid for field references.
+ * Used for primaryKey (schema allows dimension keys or attributes) and other paths (attributes only).
  *
  * @param {Object} parsed - Parsed datasource object
  * @returns {{ attributes: string[], attributesAndDimensions: Set<string> }}
  */
 function getNormalizedSets(parsed) {
   const attributes = Object.keys(parsed?.fieldMappings?.attributes ?? {});
-  const dimensions = Object.keys(parsed?.fieldMappings?.dimensions ?? {});
-  const attributesAndDimensions = new Set([...attributes, ...dimensions]);
+  const rootDims = Object.keys(parsed?.dimensions ?? {}).filter(k => {
+    const b = parsed.dimensions[k];
+    return b && typeof b === 'object';
+  });
+  const attributesAndDimensions = new Set([...attributes, ...rootDims]);
   return { attributes, attributesAndDimensions };
 }
 
@@ -31,7 +34,7 @@ function checkPrimaryKey(parsed, attributesAndDimensions, errors) {
   primaryKey.forEach((field, i) => {
     if (typeof field === 'string' && field !== '' && !attributesAndDimensions.has(field)) {
       errors.push(
-        `primaryKey[${i}]: field '${field}' does not exist in fieldMappings.attributes or fieldMappings.dimensions. Each primaryKey value must reference an attribute or dimension name.`
+        `primaryKey[${i}]: field '${field}' does not exist in fieldMappings.attributes or root dimensions. Each primaryKey value must reference an attribute or dimension key.`
       );
     }
   });
@@ -98,7 +101,7 @@ function checkIndexingAndValidation(parsed, normalizedAttributes, errors) {
 /**
  * Validates that all field references in indexing, validation, quality,
  * primaryKey, and exposed.profiles exist in fieldMappings.attributes (or
- * dimensions for primaryKey per schema). When fieldMappings.attributes is
+ * root dimension keys for primaryKey per schema). When fieldMappings.attributes is
  * missing or empty, returns no errors (skip check, matching dataplane behavior).
  *
  * @function validateFieldReferences

package/lib/datasource/integration-context.js

@@ -0,0 +1,63 @@
+/**
+ * @fileoverview Shared integration app resolution (system key, datasource file lookup).
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const path = require('path');
+const fs = require('fs').promises;
+const { getIntegrationPath } = require('../utils/paths');
+const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
+const { loadConfigFile } = require('../utils/config-format');
+
+/**
+ * @param {string} appKey - Integration app key
+ * @returns {Promise<string>} systemKey
+ */
+async function getSystemKeyFromAppKey(appKey) {
+  const appPath = getIntegrationPath(appKey);
+  const configPath = resolveApplicationConfigPath(appPath);
+  const config = loadConfigFile(configPath);
+  if (!config.externalIntegration || !config.externalIntegration.systems || config.externalIntegration.systems.length === 0) {
+    throw new Error(`No externalIntegration.systems found in ${configPath}`);
+  }
+  const systemFile = config.externalIntegration.systems[0];
+  const systemPath = path.isAbsolute(systemFile)
+    ? systemFile
+    : path.join(appPath, systemFile);
+  const systemContent = await fs.readFile(systemPath, 'utf8');
+  const yaml = require('js-yaml');
+  const systemConfig = yaml.load(systemContent);
+  return systemConfig?.key || path.basename(systemFile, '-system.yaml').replace('-system', '');
+}
+
+/**
+ * Find a datasource filename by matching the key inside the file.
+ * @param {string} appPath
+ * @param {string} schemaBasePath
+ * @param {string[]} datasourceFiles
+ * @param {string} datasourceKey
+ * @returns {string|null}
+ */
+function findDatasourceFileByKey(appPath, schemaBasePath, datasourceFiles, datasourceKey) {
+  const fsSync = require('fs');
+  for (const f of datasourceFiles) {
+    if (!f || typeof f !== 'string') continue;
+    const fullPath = path.isAbsolute(schemaBasePath)
+      ? path.join(schemaBasePath, f)
+      : path.join(appPath, schemaBasePath, f);
+    if (!fsSync.existsSync(fullPath)) continue;
+    try {
+      const parsed = loadConfigFile(fullPath);
+      if (parsed && parsed.key === datasourceKey) return f;
+    } catch {
+      // skip unreadable or invalid files
+    }
+  }
+  return null;
+}
+
+module.exports = {
+  getSystemKeyFromAppKey,
+  findDatasourceFileByKey
+};

package/lib/datasource/list.js

@@ -1,7 +1,8 @@
+const { formatBlockingError } = require('../utils/cli-test-layout-chalk');
 /**
  * Datasource List Command
  *
- * Lists datasources from the dataplane (GET /api/v1/external/).
+ * Lists datasources from the dataplane (GET /api/v1/external).
  * Resolves dataplane URL from the controller, then calls the dataplane list API.
  *
  * @fileoverview Datasource listing for AI Fabrix Builder
@@ -74,7 +75,7 @@ function extractFromPaginatedFormat(apiResponse) {
  * @param {Object} apiResponse - API response object
  */
 function logInvalidResponseError(apiResponse) {
-  logger.error(chalk.red('❌ Invalid response: expected data array or items array'));
+  logger.error(formatBlockingError('Invalid response: expected data array or items array'));
   logger.error(chalk.gray('\nAPI response type:'), typeof apiResponse);
   logger.error(chalk.gray('API response:'), JSON.stringify(apiResponse, null, 2));
 }
@@ -177,7 +178,7 @@ async function getDeviceTokenFromConfig(config) {
  */
 function validateDatasourceListingAuth(token, controllerUrl) {
   if (!token || !controllerUrl || (typeof controllerUrl === 'string' && !controllerUrl.trim())) {
-    logger.error(chalk.red('❌ Not logged in. Run: aifabrix login'));
+    logger.error(formatBlockingError('Not logged in. Run: aifabrix login'));
     logger.error(chalk.gray('   Use device code flow: aifabrix login --method device --controller <url>'));
     process.exit(1);
   }
@@ -215,7 +216,7 @@ function handleDatasourceApiError(response, dataplaneUrl = null) {
 function validateControllerUrl(controllerUrl) {
   const trimmed = controllerUrl.trim();
   if (!trimmed) {
-    logger.error(chalk.red('❌ Controller URL is empty.'));
+    logger.error(formatBlockingError('Controller URL is empty.'));
     logger.error(chalk.gray(`   Controller URL from config: ${JSON.stringify(controllerUrl)}`));
     logger.error(chalk.gray('   Run: aifabrix login --method device --controller <url>'));
     process.exit(1);
@@ -238,7 +239,7 @@ async function resolveAndValidateDataplaneUrl(controllerUrl, environment, authCo
     // discoverDataplaneUrl already logs progress and success messages
     dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig);
   } catch (error) {
-    logger.error(chalk.red('❌ Failed to resolve dataplane URL:'), error.message);
+    logger.error(formatBlockingError('Failed to resolve dataplane URL:'), error.message);
     logger.error(chalk.gray('\nThe dataplane URL is automatically discovered from the controller.'));
     logger.error(chalk.gray('If discovery fails, ensure you are logged in and the controller is accessible:'));
     logger.error(chalk.gray('   aifabrix login'));
@@ -248,7 +249,7 @@ async function resolveAndValidateDataplaneUrl(controllerUrl, environment, authCo
   }
 
   if (!dataplaneUrl || typeof dataplaneUrl !== 'string' || !dataplaneUrl.trim()) {
-    logger.error(chalk.red('❌ Dataplane URL is empty.'));
+    logger.error(formatBlockingError('Dataplane URL is empty.'));
     logger.error(chalk.gray('The dataplane URL could not be discovered from the controller.'));
     logger.error(chalk.gray('Ensure the dataplane service is registered in the controller.'));
     process.exit(1);
@@ -295,7 +296,7 @@ async function listDatasources(_options) {
   // Resolve dataplane URL first (required for list call)
   const dataplaneUrl = await resolveAndValidateDataplaneUrl(controllerUrl, environment, authConfig);
 
-  // List datasources from dataplane (GET /api/v1/external/)
+  // List datasources from dataplane (GET /api/v1/external)
   const response = await listDatasourcesFromDataplane(dataplaneUrl, authConfig);
 
   if (!response.success || !response.data) {

package/lib/datasource/log-viewer.js

@@ -12,6 +12,7 @@ const chalk = require('chalk');
 const logger = require('../utils/logger');
 const { resolveAppKeyForDatasource } = require('./resolve-app');
 const { getIntegrationPath } = require('../utils/paths');
+const { sectionTitle, headerKeyValue, formatBlockingError, successGlyph, failureGlyph } = require('../utils/cli-test-layout-chalk');
 
 /**
  * Get the path to the latest log file in a directory matching a glob-like pattern
@@ -55,62 +56,88 @@ function truncate(s, maxLen = 60) {
  * @param {Object} data - Parsed log JSON (request, response, error)
  * @param {string} [fileName] - Log file name for header
  */
-function formatE2ELog(data, fileName) {
-  logger.log(chalk.blue('\n——— E2E Log') + (fileName ? chalk.gray(` ${fileName}`) : ''));
+function logE2ERequestSection(data) {
   const req = data.request || {};
-  logger.log(chalk.cyan('Request:'));
-  logger.log(chalk.gray(`  sourceIdOrKey: ${req.sourceIdOrKey ?? '—'}`));
+  logger.log('');
+  logger.log(sectionTitle('Request:'));
+  logger.log(headerKeyValue('sourceIdOrKey:', String(req.sourceIdOrKey ?? '—')));
   if (req.includeDebug !== undefined) logger.log(chalk.gray(`  includeDebug: ${req.includeDebug}`));
   if (req.cleanup !== undefined) logger.log(chalk.gray(`  cleanup: ${req.cleanup}`));
-  if (req.primaryKeyValue !== undefined) logger.log(chalk.gray(`  primaryKeyValue: ${truncate(JSON.stringify(req.primaryKeyValue))}`));
+  if (req.primaryKeyValue !== undefined) {
+    logger.log(chalk.gray(`  primaryKeyValue: ${truncate(JSON.stringify(req.primaryKeyValue))}`));
+  }
+  return req;
+}
+
+function logE2EStepsSection(steps) {
+  if (!Array.isArray(steps) || steps.length === 0) return;
+  logger.log(sectionTitle('Steps:'));
+  for (const step of steps) {
+    const name = step.name || step.step || 'unknown';
+    const ok = step.success !== false && !step.error;
+    logger.log(`  ${ok ? successGlyph() : failureGlyph()} ${chalk.white(name)}`);
+    if (step.error) logger.log(chalk.red(`    ${step.error}`));
+    if (step.message && ok) logger.log(chalk.gray(`    ${step.message}`));
+    if ((name === 'sync' || step.step === 'sync') && step.evidence && step.evidence.jobs) {
+      for (const job of step.evidence.jobs) {
+        const audit = job.audit || {};
+        const parts = [];
+        if (job.recordsProcessed !== undefined && job.recordsProcessed !== null) parts.push(`${job.recordsProcessed} processed`);
+        if (job.totalRecords !== undefined && job.totalRecords !== null) parts.push(`total: ${job.totalRecords}`);
+        if (
+          (audit.inserted !== undefined && audit.inserted !== null) ||
+          (audit.updated !== undefined && audit.updated !== null) ||
+          (audit.deleted !== undefined && audit.deleted !== null)
+        ) {
+          parts.push(`(inserted: ${audit.inserted ?? 0}, updated: ${audit.updated ?? 0}, deleted: ${audit.deleted ?? 0})`);
+        }
+        if (parts.length) logger.log(chalk.gray(`    Job: ${parts.join(' ')}`));
+      }
+    }
+  }
+}
+
+function logE2EAuditTraceSection(req, res) {
+  if (!res.auditLog || !Array.isArray(res.auditLog) || res.auditLog.length === 0) return;
+  logger.log('');
+  logger.log(sectionTitle('CIP execution trace(s):'));
+  logger.log(chalk.gray(`${res.auditLog.length}`));
+  const baseUrl = (req.dataplaneUrl || '').toString().replace(/\/$/, '');
+  const sourceIdOrKey = req.sourceIdOrKey || '';
+  res.auditLog.slice(0, 3).forEach((trace, i) => {
+    const id = trace.executionId || trace.id || trace.traceId;
+    if (!id) return;
+    const idStr = String(id);
+    logger.log(chalk.gray(`  ${i + 1}. executionId: ${idStr}`));
+    if (baseUrl && sourceIdOrKey) {
+      const executionUrl = `${baseUrl}/api/v1/external/${sourceIdOrKey}/executions/${idStr}`;
+      logger.log(chalk.gray(`      Link: ${executionUrl}`));
+    }
+  });
+}
+
+function logE2EResponseSection(req, data) {
   if (data.error) {
-    logger.log(chalk.red('Error: ') + data.error);
+    logger.log(formatBlockingError(`Error: ${data.error}`));
     return;
   }
   const res = data.response || {};
-  logger.log(chalk.cyan('Response:'));
+  logger.log('');
+  logger.log(sectionTitle('Response:'));
   logger.log(chalk.gray(`  success: ${res.success}`));
   if (res.status) logger.log(chalk.gray(`  status: ${res.status}`));
   if (res.error) logger.log(chalk.red(`  error: ${res.error}`));
   const steps = res.steps || res.completedActions || [];
-  if (steps.length > 0) {
-    logger.log(chalk.cyan('Steps:'));
-    for (const step of steps) {
-      const name = step.name || step.step || 'unknown';
-      const ok = step.success !== false && !step.error;
-      logger.log(`  ${ok ? chalk.green('✓') : chalk.red('✗')} ${name}`);
-      if (step.error) logger.log(chalk.red(`    ${step.error}`));
-      if (step.message && ok) logger.log(chalk.gray(`    ${step.message}`));
-      if ((name === 'sync' || step.step === 'sync') && step.evidence && step.evidence.jobs) {
-        for (const job of step.evidence.jobs) {
-          const audit = job.audit || {};
-          const parts = [];
-          if (job.recordsProcessed !== undefined && job.recordsProcessed !== null) parts.push(`${job.recordsProcessed} processed`);
-          if (job.totalRecords !== undefined && job.totalRecords !== null) parts.push(`total: ${job.totalRecords}`);
-          if (audit.inserted !== undefined && audit.inserted !== null || audit.updated !== undefined && audit.updated !== null || audit.deleted !== undefined && audit.deleted !== null) {
-            parts.push(`(inserted: ${audit.inserted ?? 0}, updated: ${audit.updated ?? 0}, deleted: ${audit.deleted ?? 0})`);
-          }
-          if (parts.length) logger.log(chalk.gray(`    Job: ${parts.join(' ')}`));
-        }
-      }
-    }
-  }
-  if (res.auditLog && Array.isArray(res.auditLog) && res.auditLog.length > 0) {
-    logger.log(chalk.cyan('CIP execution trace(s): ') + chalk.gray(`${res.auditLog.length}`));
-    const baseUrl = (req.dataplaneUrl || '').toString().replace(/\/$/, '');
-    const sourceIdOrKey = req.sourceIdOrKey || '';
-    res.auditLog.slice(0, 3).forEach((trace, i) => {
-      const id = trace.executionId || trace.id || trace.traceId;
-      if (id) {
-        const idStr = String(id);
-        logger.log(chalk.gray(`  ${i + 1}. executionId: ${idStr}`));
-        if (baseUrl && sourceIdOrKey) {
-          const executionUrl = `${baseUrl}/api/v1/external/${sourceIdOrKey}/executions/${idStr}`;
-          logger.log(chalk.gray(`      Link: ${executionUrl}`));
-        }
-      }
-    });
-  }
+  logE2EStepsSection(steps);
+  logE2EAuditTraceSection(req, res);
+}
+
+function formatE2ELog(data, fileName) {
+  logger.log('');
+  logger.log(sectionTitle('E2E log'));
+  if (fileName) logger.log(chalk.gray(fileName));
+  const req = logE2ERequestSection(data);
+  logE2EResponseSection(req, data);
 }
 
 /**
@@ -119,38 +146,42 @@ function formatE2ELog(data, fileName) {
  * @param {string} [fileName] - Log file name for header
  */
 function formatIntegrationLog(data, fileName) {
-  logger.log(chalk.blue('\n——— Integration Log') + (fileName ? chalk.gray(` ${fileName}`) : ''));
+  logger.log('');
+  logger.log(sectionTitle('Integration log'));
+  if (fileName) logger.log(chalk.gray(fileName));
   const req = data.request || {};
-  logger.log(chalk.cyan('Request:'));
+  logger.log('');
+  logger.log(sectionTitle('Request:'));
   logger.log(chalk.gray(`  systemKey: ${req.systemKey ?? '—'}, datasourceKey: ${req.datasourceKey ?? '—'}`));
   if (req.includeDebug !== undefined) logger.log(chalk.gray(`  includeDebug: ${req.includeDebug}`));
   if (data.error) {
-    logger.log(chalk.red('Error: ') + data.error);
+    logger.log(formatBlockingError(`Error: ${data.error}`));
     return;
   }
   const res = data.response || {};
-  logger.log(chalk.cyan('Response:'));
+  logger.log('');
+  logger.log(sectionTitle('Response:'));
   logger.log(chalk.gray(`  success: ${res.success}`));
   if (res.error) logger.log(chalk.red(`  error: ${res.error}`));
   const vr = res.validationResults || {};
-  logger.log(chalk.cyan('Validation:'));
+  logger.log(sectionTitle('Validation:'));
   logger.log(chalk.gray(`  isValid: ${vr.isValid}`));
   if (vr.errors && vr.errors.length) vr.errors.forEach(e => logger.log(chalk.red(`    - ${e}`)));
   const fmr = res.fieldMappingResults || {};
   if (Object.keys(fmr).length) {
-    logger.log(chalk.cyan('Field mapping:'));
+    logger.log(sectionTitle('Field mapping:'));
     logger.log(chalk.gray(`  mappingCount: ${fmr.mappingCount ?? '—'}`));
     if (fmr.dimensions) logger.log(chalk.gray(`  dimensions: ${Object.keys(fmr.dimensions).join(', ')}`));
   }
   const etr = res.endpointTestResults || {};
   if (Object.keys(etr).length) {
-    logger.log(chalk.cyan('Endpoint:'));
+    logger.log(sectionTitle('Endpoint:'));
     logger.log(chalk.gray(`  endpointConfigured: ${etr.endpointConfigured}`));
   }
   if (res.normalizedOutput || res.normalizedMetadata) {
     const out = res.normalizedOutput || res.normalizedMetadata;
     const keys = typeof out === 'object' && out !== null ? Object.keys(out) : [];
-    logger.log(chalk.cyan('Normalized output: ') + chalk.gray(keys.length ? `${keys.length} fields` : '—'));
+    logger.log(sectionTitle('Normalized output:') + ' ' + chalk.gray(keys.length ? `${keys.length} fields` : '—'));
   }
 }
 

package/lib/datasource/resolve-app.js

@@ -17,7 +17,7 @@ const { loadConfigFile } = require('../utils/config-format');
 
 /**
  * For one app, check if any of its datasource files has the given key
- * @param {string} appKey - Integration app key
+ * @param {string} appKey - Integration folder name (system key context)
  * @param {string} datasourceKey - Datasource key to match
  * @returns {boolean} True if this app has a datasource with that key
  */
@@ -52,7 +52,7 @@ function appHasDatasourceKey(appKey, datasourceKey) {
  * Resolve app key for a datasource: explicit --app, cwd, scan by key, or parse key convention.
  * @async
  * @param {string} datasourceKey - Datasource key (e.g. hubspot-test-company)
- * @param {string} [explicitApp] - Explicit app key from --app
+ * @param {string} [explicitApp] - Explicit integration folder from --app
  * @returns {Promise<{appKey: string}>} Resolved app key
  * @throws {Error} When app cannot be determined or multiple apps match
  */
@@ -85,7 +85,7 @@ async function resolveAppKeyForDatasource(datasourceKey, explicitApp) {
   }
   if (matches.length > 1) {
     throw new Error(
-      `More than one app has this datasource; add --app <appKey>. Apps: ${matches.join(', ')}`
+      `More than one app has this datasource; add --app <app>. Apps: ${matches.join(', ')}`
     );
   }
 
@@ -99,7 +99,7 @@ async function resolveAppKeyForDatasource(datasourceKey, explicitApp) {
   }
 
   throw new Error(
-    'Could not determine app context. Use --app <appKey> or run from integration/<appKey>/ directory.'
+    'Could not determine app context. Use --app <app> or run from integration/<systemKey>/ directory.'
   );
 }