@aifabrix/builder 2.45.0 → 2.46.0

package/lib/datasource/resolve-app.js

@@ -5,49 +5,82 @@
  * @version 2.0.0
  */
 
-const path = require('path');
 const fs = require('fs');
 const {
   getIntegrationPath,
   listIntegrationAppNames,
   resolveIntegrationAppKeyFromCwd
 } = require('../utils/paths');
-const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
-const { loadConfigFile } = require('../utils/config-format');
+const {
+  appHasDatasourceIdentifier,
+  pickIntegrationAppForDatasourceIdentifier
+} = require('./datasource-identifier');
+
+/** @deprecated Use appHasDatasourceIdentifier — kept for tests and backward compatibility */
+function appHasDatasourceKey(appKey, datasourceKey) {
+  return appHasDatasourceIdentifier(appKey, datasourceKey);
+}
 
 /**
- * For one app, check if any of its datasource files has the given key
- * @param {string} appKey - Integration folder name (system key context)
- * @param {string} datasourceKey - Datasource key to match
- * @returns {boolean} True if this app has a datasource with that key
+ * @param {string} key
+ * @param {string} [explicitApp]
+ * @returns {{ appKey: string }|null}
  */
-function appHasDatasourceKey(appKey, datasourceKey) {
-  const appPath = getIntegrationPath(appKey);
-  let config;
-  try {
-    const configPath = resolveApplicationConfigPath(appPath);
-    config = loadConfigFile(configPath);
-  } catch {
-    return false;
+function resolveFromExplicitApp(key, explicitApp) {
+  if (!explicitApp || typeof explicitApp !== 'string' || !explicitApp.trim()) {
+    return null;
+  }
+  const appKey = explicitApp.trim();
+  if (fs.existsSync(getIntegrationPath(appKey))) {
+    return { appKey };
+  }
+  return null;
+}
+
+/**
+ * @param {string} key
+ * @param {string[]} appNames
+ * @returns {{ appKey: string }|null}
+ * @throws {Error}
+ */
+function resolveFromAppScan(key, appNames) {
+  const matches = appNames.filter(appName => appHasDatasourceIdentifier(appName, key));
+  if (matches.length === 1) {
+    return { appKey: matches[0] };
+  }
+  if (matches.length > 1) {
+    throw new Error(
+      `More than one app has this datasource; add --app <app>. Apps: ${matches.join(', ')}`
+    );
   }
-  const schemaBasePath = config.externalIntegration?.schemaBasePath || './';
-  const datasourceFiles = config.externalIntegration?.dataSources || [];
-  for (const f of datasourceFiles) {
-    if (!f || typeof f !== 'string') continue;
-    const fullPath = path.isAbsolute(schemaBasePath)
-      ? path.join(schemaBasePath, f)
-      : path.join(appPath, schemaBasePath, f);
-    if (!fs.existsSync(fullPath)) continue;
-    try {
-      const parsed = loadConfigFile(fullPath);
-      if (parsed && parsed.key === datasourceKey) return true;
-    } catch {
-      // skip unreadable or invalid files
-    }
+  return null;
+}
+
+/**
+ * @param {string} key
+ * @param {string[]} appNames
+ * @returns {{ appKey: string }|null}
+ */
+function resolveFromPrefixOrSegments(key, appNames) {
+  const prefixApp = pickIntegrationAppForDatasourceIdentifier(key, appNames);
+  if (prefixApp && appHasDatasourceIdentifier(prefixApp, key)) {
+    return { appKey: prefixApp };
   }
-  return false;
+  const segments = key.split('-');
+  if (segments.length < 2) {
+    return null;
+  }
+  const candidate = segments.slice(0, -1).join('-');
+  const candidatePath = getIntegrationPath(candidate);
+  if (fs.existsSync(candidatePath) && appHasDatasourceIdentifier(candidate, key)) {
+    return { appKey: candidate };
+  }
+  return null;
 }
 
+const RESOLVE_APP_HINT =
+  'Could not determine app context. Use --app <app>, a datasource key, a datasource JSON filename (e.g. myapp-datasource-contacts), or run from integration/<systemKey>/ directory.';
+
 /**
  * Resolve app key for a datasource: explicit --app, cwd, scan by key, or parse key convention.
  * @async
@@ -56,7 +89,6 @@ function appHasDatasourceKey(appKey, datasourceKey) {
  * @returns {Promise<{appKey: string}>} Resolved app key
  * @throws {Error} When app cannot be determined or multiple apps match
  */
-/* eslint-disable-next-line max-statements -- Resolution order: explicit, cwd, scan, parse */
 async function resolveAppKeyForDatasource(datasourceKey, explicitApp) {
   if (!datasourceKey || typeof datasourceKey !== 'string') {
     throw new Error('Datasource key is required');
@@ -66,41 +98,28 @@ async function resolveAppKeyForDatasource(datasourceKey, explicitApp) {
     throw new Error('Datasource key cannot be empty');
   }
 
-  if (explicitApp && typeof explicitApp === 'string' && explicitApp.trim()) {
-    const appPath = getIntegrationPath(explicitApp.trim());
-    if (fs.existsSync(appPath)) {
-      return { appKey: explicitApp.trim() };
-    }
+  const explicit = resolveFromExplicitApp(key, explicitApp);
+  if (explicit) {
+    return explicit;
   }
 
   const fromCwd = resolveIntegrationAppKeyFromCwd();
-  if (fromCwd && appHasDatasourceKey(fromCwd, key)) {
+  if (fromCwd && appHasDatasourceIdentifier(fromCwd, key)) {
     return { appKey: fromCwd };
   }
 
   const appNames = listIntegrationAppNames();
-  const matches = appNames.filter(appName => appHasDatasourceKey(appName, key));
-  if (matches.length === 1) {
-    return { appKey: matches[0] };
-  }
-  if (matches.length > 1) {
-    throw new Error(
-      `More than one app has this datasource; add --app <app>. Apps: ${matches.join(', ')}`
-    );
+  const scanned = resolveFromAppScan(key, appNames);
+  if (scanned) {
+    return scanned;
   }
 
-  const segments = key.split('-');
-  if (segments.length >= 2) {
-    const candidate = segments.slice(0, -1).join('-');
-    const candidatePath = getIntegrationPath(candidate);
-    if (fs.existsSync(candidatePath) && appHasDatasourceKey(candidate, key)) {
-      return { appKey: candidate };
-    }
+  const inferred = resolveFromPrefixOrSegments(key, appNames);
+  if (inferred) {
+    return inferred;
   }
 
-  throw new Error(
-    'Could not determine app context. Use --app <app> or run from integration/<systemKey>/ directory.'
-  );
+  throw new Error(RESOLVE_APP_HINT);
 }
 
 module.exports = {

package/lib/datasource/test-e2e.js

@@ -11,7 +11,7 @@ const chalk = require('chalk');
 const logger = require('../utils/logger');
 const { getIntegrationPath } = require('../utils/paths');
 const { resolveAppKeyForDatasource } = require('./resolve-app');
-const { infoLine } = require('../utils/cli-test-layout-chalk');
+const { formatProgress, formatSuccessLine, infoLine } = require('../utils/cli-test-layout-chalk');
 const { runUnifiedDatasourceValidation } = require('./unified-validation-run');
 const { includeDebugForRequest } = require('../utils/validation-run-request');
 const { e2eShapeFromEnvelope } = require('../utils/datasource-test-run-legacy-adapter');
@@ -41,7 +41,12 @@ function buildUnifiedE2eRunOptions(options, timeoutMs, pk) {
     minRecordCount: options.minRecordCount,
     capabilityKey: options.capabilityKey,
     timeout: timeoutMs,
-    sync: options.sync === true
+    sync: options.sync === true,
+    force: options.force === true,
+    authConfig: options.authConfig,
+    dataplaneUrl: options.dataplaneUrl || options.dataplane,
+    quietPollUi: options.quietPollUi === true,
+    onPollProgress: options.onPollProgress
   };
 }
 
@@ -51,6 +56,25 @@ function logE2eDatasourceBanner(datasourceKey, verbose) {
   logger.log(infoLine(`🧪 Running E2E test for datasource: ${datasourceKey}`));
 }
 
+/**
+ * Compact per-datasource progress (default test-e2e / system rollup when not --verbose).
+ * @param {string} datasourceKey
+ * @param {boolean} verbose
+ */
+function logE2eDatasourceProgress(datasourceKey, verbose) {
+  if (verbose) return;
+  logger.log(formatProgress(`Running E2E validation for ${datasourceKey}…`));
+}
+
+/**
+ * @param {string} datasourceKey
+ * @param {boolean} verbose
+ */
+function logE2eDatasourceValidated(datasourceKey, verbose) {
+  if (verbose) return;
+  logger.log(formatSuccessLine(`${datasourceKey} validated`));
+}
+
 /**
  * Resolve primaryKeyValue for request body: string as-is, or read and parse JSON from @path
  * @param {string} [value]
@@ -174,6 +198,7 @@ async function runDatasourceTestE2E(datasourceKey, options = {}) {
   const cliWallStartedAt = Date.now();
   const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
   logE2eDatasourceBanner(datasourceKey, options.verbose);
+  logE2eDatasourceProgress(datasourceKey, options.verbose);
   const pk = await resolvePrimaryKeyValue(options.primaryKeyValue);
   const timeoutMs = resolveE2ePollTimeoutMs(options);
   const requestMeta = buildE2eRequestMeta(datasourceKey, options, pk);
@@ -184,6 +209,8 @@ async function runDatasourceTestE2E(datasourceKey, options = {}) {
 
   await throwIfUnifiedE2EBlocked(unifiedResult, appKey, options, requestMeta);
 
+  logE2eDatasourceValidated(datasourceKey, options.verbose);
+
   const display = e2eShapeFromEnvelope(unifiedResult.envelope);
   Object.assign(display, { datasourceTestRun: unifiedResult.envelope });
 

package/lib/datasource/unified-validation-run-body.js

@@ -10,6 +10,7 @@ const {
   includeDebugForRequest
 } = require('../utils/validation-run-request');
 const testHelpers = require('../utils/external-system-test-helpers');
+const { withStageOnValidationRunBody } = require('../utils/validation-run-stage');
 
 /**
  * @param {Object} params
@@ -52,7 +53,7 @@ async function buildUnifiedValidationBody(params) {
       })
       : undefined;
 
-  return buildExternalDataSourceValidationRequest({
+  const body = buildExternalDataSourceValidationRequest({
     systemKey,
     datasourceKey,
     runType,
@@ -62,6 +63,7 @@ async function buildUnifiedValidationBody(params) {
     explain: options.verbose === true,
     e2eOptions: e2eOptions && Object.keys(e2eOptions).length > 0 ? e2eOptions : undefined
   });
+  return withStageOnValidationRunBody(body, runType);
 }
 
 module.exports = { buildUnifiedValidationBody };

package/lib/datasource/unified-validation-run-resolve.js

@@ -4,40 +4,73 @@
  * @version 2.0.0
  */
 
-const path = require('path');
 const { getIntegrationPath } = require('../utils/paths');
 const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
 const { loadConfigFile } = require('../utils/config-format');
+const {
+  datasourceIdentifierMatchesFile,
+  resolveDatasourceFilePath
+} = require('./datasource-identifier');
+
+/**
+ * @param {string} appPath
+ * @param {string} schemaBasePath
+ * @param {string[]} datasourceFiles
+ * @param {string} identifier
+ * @returns {string|null}
+ */
+function findDatasourceFileByIdentifier(appPath, schemaBasePath, datasourceFiles, identifier) {
+  const fsSync = require('fs');
+  for (const fileRef of datasourceFiles) {
+    if (!fileRef || typeof fileRef !== 'string') {
+      continue;
+    }
+    const fullPath = resolveDatasourceFilePath(appPath, schemaBasePath, fileRef);
+    if (!fsSync.existsSync(fullPath)) {
+      continue;
+    }
+    try {
+      const parsed = loadConfigFile(fullPath);
+      if (datasourceIdentifierMatchesFile(identifier, fileRef, parsed)) {
+        return fileRef;
+      }
+    } catch {
+      // skip unreadable or invalid files
+    }
+  }
+  return null;
+}
 
 /**
  * @param {string} appKey
  * @param {string} datasourceKey
  * @param {Function} findDatasourceFileByKey - From integration-context
- * @returns {{ datasource: Object, datasourcePath: string }}
+ * @returns {{ datasource: Object, datasourcePath: string, datasourceKey: string }}
  */
 function loadDatasourceForApp(appKey, datasourceKey, findDatasourceFileByKey) {
   const appPath = getIntegrationPath(appKey);
   const config = loadConfigFile(resolveApplicationConfigPath(appPath));
   const schemaBasePath = config.externalIntegration?.schemaBasePath || './';
   const datasourceFiles = config.externalIntegration?.dataSources || [];
-  let datasourceFile = datasourceFiles.find(f => {
-    const base = path.basename(f, path.extname(f));
-    return base === datasourceKey || base.includes(datasourceKey);
-  });
+  let datasourceFile = findDatasourceFileByIdentifier(
+    appPath,
+    schemaBasePath,
+    datasourceFiles,
+    datasourceKey
+  );
   if (!datasourceFile) {
     datasourceFile = findDatasourceFileByKey(appPath, schemaBasePath, datasourceFiles, datasourceKey);
   }
   if (!datasourceFile) {
     throw new Error(`Datasource '${datasourceKey}' not found in application config`);
   }
-  const datasourcePath = path.isAbsolute(schemaBasePath)
-    ? path.join(schemaBasePath, datasourceFile)
-    : path.join(appPath, schemaBasePath, datasourceFile);
+  const datasourcePath = resolveDatasourceFilePath(appPath, schemaBasePath, datasourceFile);
   const datasource = loadConfigFile(datasourcePath);
-  if (datasource.key !== datasourceKey) {
-    throw new Error(`Datasource key mismatch: file has '${datasource.key}', expected '${datasourceKey}'`);
+  const canonicalKey = datasource.key;
+  if (!canonicalKey || typeof canonicalKey !== 'string') {
+    throw new Error(`Datasource file is missing a valid key: ${datasourcePath}`);
   }
-  return { datasource, datasourcePath };
+  return { datasource, datasourcePath, datasourceKey: canonicalKey };
 }
 
 module.exports = { loadDatasourceForApp };

package/lib/datasource/unified-validation-run.js

@@ -18,6 +18,7 @@ const { buildUnifiedValidationBody } = require('./unified-validation-run-body');
 const { postValidationRunAndOptionalPoll } = require('./unified-validation-run-post');
 const { requireBearerForDataplanePipeline } = require('../utils/token-manager');
 const logger = require('../utils/logger');
+const { buildMinimalUploadSyncOptions } = require('../utils/upload-sync-options');
 
 function _resourceTypeFromDatasource(datasource) {
   return datasource && typeof datasource === 'object' && typeof datasource.resourceType === 'string'
@@ -59,7 +60,7 @@ async function maybeSyncDatasourceToDataplane({ options, authConfig, systemKey }
   // are updated before validation runs. Datasource-only publish can still fail DP-SEC-013 when the
   // dataplane system permissions[] are stale.
   const { uploadExternalSystem } = require('../commands/upload');
-  await uploadExternalSystem(systemKey, { minimal: true });
+  await uploadExternalSystem(systemKey, buildMinimalUploadSyncOptions(options));
   logger.log(formatSuccessLine('Sync complete'));
 }
 
@@ -71,6 +72,40 @@ function loadDatasourceForAppExport(appKey, datasourceKey) {
   return loadDatasourceForApp(appKey, datasourceKey, findDatasourceFileByKey);
 }
 
+/**
+ * @async
+ * @param {string} appKey
+ * @param {Object} options
+ * @returns {Promise<{ authConfig: Object, dataplaneUrl: string }>}
+ */
+async function resolveValidationAuth(appKey, options) {
+  if (options.authConfig && (options.dataplaneUrl || options.dataplane)) {
+    return {
+      authConfig: options.authConfig,
+      dataplaneUrl: options.dataplaneUrl || options.dataplane
+    };
+  }
+  const configObj = await getConfig();
+  const auth = await setupIntegrationTestAuth(appKey, options, configObj);
+  return { authConfig: auth.authConfig, dataplaneUrl: auth.dataplaneUrl };
+}
+
+/**
+ * @async
+ * @param {string} datasourceKey
+ * @param {Object} options
+ * @returns {Promise<Object>}
+ */
+async function prepareValidationRunContext(datasourceKey, options) {
+  const runType = options.runType || 'test';
+  const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
+  const systemKey = await getSystemKeyFromAppKey(appKey);
+  const { datasource } = loadDatasourceForAppExport(appKey, datasourceKey);
+  const { authConfig, dataplaneUrl } = await resolveValidationAuth(appKey, options);
+  await maybeSyncDatasourceToDataplane({ options, authConfig, systemKey });
+  return { runType, systemKey, datasource, authConfig, dataplaneUrl };
+}
+
 /**
  * Run unified validation for one persisted datasource.
  * @async
@@ -83,41 +118,35 @@ async function runUnifiedDatasourceValidation(datasourceKey, options) {
   if (!datasourceKey || typeof datasourceKey !== 'string') {
     throw new Error('Datasource key is required');
   }
-  const runType = options.runType || 'test';
-  const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
-  const systemKey = await getSystemKeyFromAppKey(appKey);
-  const { datasource } = loadDatasourceForAppExport(appKey, datasourceKey);
-
-  const configObj = await getConfig();
-  const { authConfig, dataplaneUrl } = await setupIntegrationTestAuth(appKey, options, configObj);
-
-  await maybeSyncDatasourceToDataplane({ options, authConfig, systemKey });
 
+  const ctx = await prepareValidationRunContext(datasourceKey, options);
   const useAsync = options.noAsync ? false : options.async !== false;
   const timeoutMs = parseInt(String(options.timeout || '30000'), 10) || 30000;
-
   const body = await buildUnifiedValidationBody({
-    systemKey,
+    systemKey: ctx.systemKey,
     datasourceKey,
-    runType,
-    datasource,
+    runType: ctx.runType,
+    datasource: ctx.datasource,
     payloadPath: options.payload,
     useAsync,
     options
   });
 
   const result = await postValidationRunAndOptionalPoll({
-    dataplaneUrl,
-    authConfig,
+    dataplaneUrl: ctx.dataplaneUrl,
+    authConfig: ctx.authConfig,
     body,
     timeoutMs,
     useAsync,
     noAsync: options.noAsync === true || options.async === false,
-    verbosePoll: options.verbose === true
+    verbosePoll: options.verbose === true,
+    quietPollUi: options.quietPollUi === true,
+    pollSubject: datasourceKey,
+    onPollProgress: options.onPollProgress
   });
 
   if (result && result.envelope) {
-    annotateMissingAutoRbacPermissions(result.envelope, datasourceKey, datasource);
+    annotateMissingAutoRbacPermissions(result.envelope, datasourceKey, ctx.datasource);
   }
   return result;
 }

package/lib/datasource/validate.js

@@ -11,6 +11,11 @@
 const path = require('path');
 const fs = require('fs');
 const pathsUtil = require('../utils/paths');
+const {
+  datasourceIdentifierMatchesFile,
+  pickIntegrationAppForDatasourceIdentifier,
+  collectDatasourceFileRefsMatchingIdentifier
+} = require('./datasource-identifier');
 const { loadExternalDataSourceSchema } = require('../utils/schema-loader');
 const { formatValidationErrors } = require('../utils/error-formatter');
 const { validateFieldReferences } = require('./field-reference-validator');
@@ -46,16 +51,13 @@ function tryResolveAsExistingFile(identifier) {
 }
 
 /**
- * Pick integration app folder for a datasource `key` (longest matching prefix wins).
+ * Pick integration app folder for a datasource identifier (longest matching prefix wins).
  * @param {string} datasourceKey
  * @param {string[]} appNames
  * @returns {string|null}
  */
 function pickIntegrationAppForDatasourceKey(datasourceKey, appNames) {
-  const matches = appNames.filter((app) => datasourceKey === app || datasourceKey.startsWith(`${app}-`));
-  if (matches.length === 0) return null;
-  matches.sort((a, b) => b.length - a.length);
-  return matches[0];
+  return pickIntegrationAppForDatasourceIdentifier(datasourceKey, appNames);
 }
 
 /**
@@ -94,7 +96,7 @@ function collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey) {
     } catch {
       continue;
     }
-    if (parsed && typeof parsed === 'object' && parsed.key === datasourceKey) {
+    if (datasourceIdentifierMatchesFile(datasourceKey, name, parsed)) {
       hits.push(full);
     }
   }
@@ -102,21 +104,12 @@ function collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey) {
 }
 
 /**
- * Find integration JSON whose parsed `key` equals datasourceKey.
+ * @param {string} appName
  * @param {string} datasourceKey
- * @returns {string} Absolute file path
+ * @returns {string}
  * @throws {Error}
  */
-function resolvePathFromIntegrationDatasourceKey(datasourceKey) {
-  const appNames = pathsUtil.listIntegrationAppNames();
-  const appName = pickIntegrationAppForDatasourceKey(datasourceKey, appNames);
-  if (!appName) {
-    throw new Error(
-      `No integration/<app>/ folder matches datasource key "${datasourceKey}". ` +
-        'Pass a path to the JSON file, or use a key whose prefix matches an app under integration/ ' +
-        '(e.g. app test-e2e-hubspot for key test-e2e-hubspot-users).'
-    );
-  }
+function resolvePathFromIntegrationFolderScan(appName, datasourceKey) {
   const dir = pathsUtil.getIntegrationPath(appName);
   if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
     throw new Error(`Integration folder not found: ${dir}`);
@@ -128,16 +121,40 @@ function resolvePathFromIntegrationDatasourceKey(datasourceKey) {
   const hits = collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey);
   if (hits.length === 0) {
     throw new Error(
-      `No datasource JSON with key "${datasourceKey}" under integration/${appName}/. ` +
+      `No datasource JSON matching "${datasourceKey}" under integration/${appName}/. ` +
         `Checked: ${scanOrder.length ? scanOrder.join(', ') : '(no JSON candidates)'}`
     );
   }
   if (hits.length > 1) {
-    throw new Error(`Ambiguous: multiple files declare key "${datasourceKey}": ${hits.join(', ')}`);
+    throw new Error(`Ambiguous: multiple files match "${datasourceKey}": ${hits.join(', ')}`);
   }
   return hits[0];
 }
 
+/**
+ * Find integration JSON whose parsed `key` equals datasourceKey.
+ * @param {string} datasourceKey
+ * @returns {string} Absolute file path
+ * @throws {Error}
+ */
+function resolvePathFromIntegrationDatasourceKey(datasourceKey) {
+  const appNames = pathsUtil.listIntegrationAppNames();
+  const appName = pickIntegrationAppForDatasourceKey(datasourceKey, appNames);
+  if (!appName) {
+    throw new Error(
+      `No integration/<app>/ folder matches datasource identifier "${datasourceKey}". ` +
+        'Pass a path to the JSON file, or use a datasource key or filename whose prefix matches an app under integration/ ' +
+        '(e.g. test-e2e-hubspot-companies or test-e2e-hubspot-datasource-companies for app test-e2e-hubspot).'
+    );
+  }
+  const manifestHits = collectDatasourceFileRefsMatchingIdentifier(appName, datasourceKey);
+  if (manifestHits.length === 1) {
+    const dir = pathsUtil.getIntegrationPath(appName);
+    return path.join(dir, manifestHits[0]);
+  }
+  return resolvePathFromIntegrationFolderScan(appName, datasourceKey);
+}
+
 /**
  * @param {string} identifier - File path or datasource `key`
  * @returns {string} Absolute path to JSON file