@aifabrix/builder 2.41.0 → 2.42.0

package/lib/commands/credential-list.js

@@ -1,7 +1,6 @@
 /**
  * Credential list command – list credentials from Dataplane
  * GET /api/v1/credential. Used by `aifabrix credential list`.
- * The Controller does not expose this endpoint; credentials are listed from the Dataplane.
  *
  * @fileoverview Credential list command implementation
  * @author AI Fabrix Team
@@ -10,6 +9,7 @@
 
 const chalk = require('chalk');
 const logger = require('../utils/logger');
+const { formatCredentialWithStatus } = require('../utils/credential-display');
 const { resolveControllerUrl } = require('../utils/controller-url');
 const { getOrRefreshDeviceToken } = require('../utils/token-manager');
 const { normalizeControllerUrl, resolveEnvironment } = require('../core/config');
@@ -37,12 +37,14 @@ async function getCredentialListAuth(controllerUrl) {
 }
 
 /**
- * Extract credentials array from API response
- * @param {Object} response - API response
+ * Extract credentials array from API response.
+ * Handles: { data: { meta, data: [...] } } (paginated), { data: [...] }, { credentials: [...] }, { items: [...] }.
+ * @param {Object} response - API response (e.g. { success, data } from API client, or raw body)
  * @returns {Array}
  */
 function extractCredentials(response) {
-  const data = response?.data ?? response;
+  const body = response?.data ?? response;
+  const data = body?.data ?? body;
   const items = data?.credentials ?? data?.items ?? (Array.isArray(data) ? data : []);
   return Array.isArray(items) ? items : [];
 }
@@ -59,9 +61,10 @@ function displayCredentialList(list, baseUrl) {
     return;
   }
   list.forEach((c) => {
-    const key = c.key ?? c.id ?? c.credentialKey ?? '-';
-    const name = c.displayName ?? c.name ?? key;
-    logger.log(`  ${chalk.cyan(key)} - ${name}`);
+    const { key, name, statusFormatted, statusLabel } = formatCredentialWithStatus(c);
+    const prefix = statusFormatted ? `${statusFormatted} ` : '  ';
+    const line = `${prefix}${chalk.cyan(key)} - ${name}${statusLabel}`;
+    logger.log(line);
   });
   logger.log('');
 }
@@ -69,10 +72,10 @@ function displayCredentialList(list, baseUrl) {
 /**
  * Ensure controller URL and auth; exit on failure. Returns { controllerUrl, authConfig } when valid.
  * @async
- * @param {Object} options - CLI options with optional controller
+ * @param {Object} options - CLI options
  * @returns {Promise<{controllerUrl: string, authConfig: Object}>}
  */
-async function ensureControllerAndAuth(options) {
+async function ensureControllerAndAuth(options = {}) {
   const controllerUrl = options.controller || (await resolveControllerUrl());
   if (!controllerUrl) {
     logger.error(chalk.red('❌ Controller URL is required. Run "aifabrix login" first.'));
@@ -91,19 +94,14 @@ async function ensureControllerAndAuth(options) {
 }
 
 /**
- * Resolve Dataplane URL for credential list (override or discover from controller + environment)
+ * Resolve Dataplane URL for credential list (discover from controller + environment)
  * @async
  * @param {string} controllerUrl - Controller base URL
  * @param {Object} authConfig - Auth config with token
- * @param {Object} options - CLI options
- * @param {string} [options.dataplane] - Optional Dataplane URL override
  * @returns {Promise<string>} Dataplane base URL
  * @throws {Error} When resolution fails (caller should exit)
  */
-async function resolveCredentialListDataplaneUrl(controllerUrl, authConfig, options) {
-  if (options.dataplane) {
-    return options.dataplane.replace(/\/$/, '');
-  }
+async function resolveCredentialListDataplaneUrl(controllerUrl, authConfig) {
   const environment = await resolveEnvironment();
   return await resolveDataplaneUrl(controllerUrl, environment, authConfig);
 }
@@ -127,12 +125,11 @@ async function fetchAndDisplayCredentials(dataplaneUrl, authConfig, listOptions)
  * @param {Object} options - CLI options
  * @returns {Promise<string>} Dataplane URL (never returns on failure; process.exit(1))
  */
-async function resolveDataplaneUrlOrExit(controllerUrl, authConfig, options) {
+async function resolveDataplaneUrlOrExit(controllerUrl, authConfig) {
   try {
-    return await resolveCredentialListDataplaneUrl(controllerUrl, authConfig, options);
+    return await resolveCredentialListDataplaneUrl(controllerUrl, authConfig);
   } catch (err) {
     logger.error(chalk.red(`❌ Could not resolve Dataplane URL: ${err.message}`));
-    logger.error(chalk.gray('Use --dataplane <url> to specify the Dataplane URL directly.'));
     process.exit(1);
   }
 }
@@ -141,15 +138,13 @@ async function resolveDataplaneUrlOrExit(controllerUrl, authConfig, options) {
  * Run credential list command: call GET /api/v1/credential on Dataplane and display results
  * @async
  * @param {Object} options - CLI options
- * @param {string} [options.controller] - Controller URL override
- * @param {string} [options.dataplane] - Dataplane URL override (default: resolved from controller + environment)
  * @param {boolean} [options.activeOnly] - List only active credentials
  * @param {number} [options.pageSize] - Items per page
  * @returns {Promise<void>}
  */
 async function runCredentialList(options = {}) {
   const { controllerUrl, authConfig } = await ensureControllerAndAuth(options);
-  const dataplaneUrl = await resolveDataplaneUrlOrExit(controllerUrl, authConfig, options);
+  const dataplaneUrl = await resolveDataplaneUrlOrExit(controllerUrl, authConfig);
   const listOptions = {
     pageSize: options.pageSize || DEFAULT_PAGE_SIZE,
     activeOnly: options.activeOnly

package/lib/commands/credential-push.js

@@ -0,0 +1,96 @@
+/**
+ * Credential push command – pushes KV_* from .env to dataplane.
+ * Used by `aifabrix credential push <system-key>`.
+ *
+ * @fileoverview Credential push command – push credential secrets to dataplane
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const path = require('path');
+const chalk = require('chalk');
+const logger = require('../utils/logger');
+const { resolveControllerUrl } = require('../utils/controller-url');
+const { getDeploymentAuth, requireBearerForDataplanePipeline } = require('../utils/token-manager');
+const { resolveDataplaneUrl } = require('../utils/dataplane-resolver');
+const { getIntegrationPath } = require('../utils/paths');
+const { pushCredentialSecrets } = require('../utils/credential-secrets-env');
+const { generateControllerManifest } = require('../generator/external-controller-manifest');
+
+/**
+ * Validates system-key format.
+ * @param {string} systemKey - System key
+ * @throws {Error} If invalid
+ */
+function validateSystemKeyFormat(systemKey) {
+  if (!systemKey || typeof systemKey !== 'string') {
+    throw new Error('System key is required and must be a string');
+  }
+  if (!/^[a-z0-9-_]+$/.test(systemKey)) {
+    throw new Error('System key must contain only lowercase letters, numbers, hyphens, and underscores');
+  }
+}
+
+/**
+ * Builds upload payload for credential push (same shape as upload).
+ * @param {string} systemKey - System key
+ * @returns {Promise<Object>} { version, application, dataSources }
+ */
+async function buildPayload(systemKey) {
+  const manifest = await generateControllerManifest(systemKey, { type: 'external' });
+  return {
+    version: manifest.version || '1.0.0',
+    application: manifest.system,
+    dataSources: manifest.dataSources || []
+  };
+}
+
+/**
+ * Runs credential push: push KV_* from .env to dataplane.
+ * @async
+ * @param {string} systemKey - External system key
+ * @returns {Promise<{ pushed: number }>} Count of secrets pushed
+ * @throws {Error} If auth or push fails
+ */
+function logPushResult(pushResult) {
+  if (pushResult.pushed > 0) {
+    const keyList = pushResult.keys?.length ? ` (${pushResult.keys.join(', ')})` : '';
+    logger.log(chalk.green(`✓ Pushed ${pushResult.pushed} credential secret(s) to dataplane${keyList}.`));
+  } else if (pushResult.skipped) {
+    logger.log(chalk.yellow('No credential secrets to push (empty .env or no KV_* vars with values).'));
+  } else {
+    logger.log(chalk.yellow('Secret push skipped'));
+  }
+  if (pushResult.warning) logger.log(chalk.yellow(`Warning: ${pushResult.warning}`));
+}
+
+async function runCredentialPush(systemKey) {
+  validateSystemKeyFormat(systemKey);
+  const appPath = getIntegrationPath(systemKey);
+  const envFilePath = path.join(appPath, '.env');
+
+  const { resolveEnvironment } = require('../core/config');
+  const environment = await resolveEnvironment();
+  const controllerUrl = await resolveControllerUrl();
+  const authConfig = await getDeploymentAuth(controllerUrl, environment, systemKey);
+
+  if (!authConfig.token && !authConfig.clientId) {
+    throw new Error('Authentication required. Run "aifabrix login" or "aifabrix app register <system-key>" first.');
+  }
+
+  requireBearerForDataplanePipeline(authConfig);
+  logger.log(chalk.blue('Resolving dataplane URL...'));
+  const dataplaneUrl = await resolveDataplaneUrl(controllerUrl, environment, authConfig);
+
+  const payload = await buildPayload(systemKey);
+  const pushResult = await pushCredentialSecrets(dataplaneUrl, authConfig, {
+    envFilePath,
+    appName: systemKey,
+    payload
+  });
+
+  logPushResult(pushResult);
+  return { pushed: pushResult.pushed || 0 };
+}
+
+module.exports = { runCredentialPush, validateSystemKeyFormat, buildPayload };

package/lib/commands/datasource.js

@@ -2,7 +2,7 @@
  * AI Fabrix Builder - Datasource Commands
  *
  * Handles datasource validation, listing, comparison, and deployment
- * Commands: datasource validate, datasource list, datasource diff, datasource deploy
+ * Commands: datasource validate, datasource list, datasource diff, datasource upload
  *
  * @fileoverview Datasource management commands for AI Fabrix Builder
  * @author AI Fabrix Team
@@ -15,6 +15,9 @@ const { validateDatasourceFile } = require('../datasource/validate');
 const { listDatasources } = require('../datasource/list');
 const { compareDatasources } = require('../datasource/diff');
 const { deployDatasource } = require('../datasource/deploy');
+const { runDatasourceTestIntegration } = require('../datasource/test-integration');
+const { runDatasourceTestE2E } = require('../datasource/test-e2e');
+const { displayIntegrationTestResults, displayE2EResults } = require('../utils/external-system-display');
 
 function setupDatasourceValidateCommand(datasource) {
   datasource.command('validate <file>')
@@ -62,14 +65,80 @@ function setupDatasourceDiffCommand(datasource) {
     });
 }
 
-function setupDatasourceDeployCommand(datasource) {
-  datasource.command('deploy <myapp> <file>')
-    .description('Deploy datasource to dataplane')
+function setupDatasourceUploadCommand(datasource) {
+  datasource.command('upload <myapp> <file>')
+    .description('Upload datasource to dataplane')
     .action(async(myapp, file, options) => {
       try {
         await deployDatasource(myapp, file, options);
       } catch (error) {
-        logger.error(chalk.red('❌ Deployment failed:'), error.message);
+        logger.error(chalk.red('❌ Upload failed:'), error.message);
+        process.exit(1);
+      }
+    });
+}
+
+function setupDatasourceTestIntegrationCommand(datasource) {
+  datasource.command('test-integration <datasourceKey>')
+    .description('Run integration (config) test for one datasource via dataplane pipeline')
+    .option('-a, --app <appKey>', 'App key (default: resolve from cwd if inside integration/<appKey>/)')
+    .option('-p, --payload <file>', 'Path to custom test payload file')
+    .option('-e, --env <env>', 'Environment: dev, tst, or pro')
+    .option('--debug', 'Include debug output and write log to integration/<appKey>/logs/')
+    .option('--timeout <ms>', 'Request timeout in milliseconds', '30000')
+    .action(async(datasourceKey, options) => {
+      try {
+        const result = await runDatasourceTestIntegration(datasourceKey, {
+          app: options.app,
+          payload: options.payload,
+          environment: options.env,
+          debug: options.debug,
+          timeout: options.timeout
+        });
+        displayIntegrationTestResults({
+          systemKey: result.systemKey || 'unknown',
+          datasourceResults: [result],
+          success: result.success
+        }, options.verbose);
+        if (!result.success) process.exit(1);
+      } catch (error) {
+        logger.error(chalk.red('❌ Integration test failed:'), error.message);
+        process.exit(1);
+      }
+    });
+}
+
+function setupDatasourceTestE2ECommand(datasource) {
+  datasource.command('test-e2e <datasourceKey>')
+    .description('Run E2E test for one datasource (config, credential, sync, data, CIP) via dataplane')
+    .option('-a, --app <appKey>', 'App key (default: resolve from cwd if inside integration/<appKey>/)')
+    .option('-e, --env <env>', 'Environment: dev, tst, or pro')
+    .option('-v, --verbose', 'Show detailed step output and poll progress')
+    .option('--debug', 'Include debug output and write log to integration/<appKey>/logs/')
+    .option('--test-crud', 'Enable CRUD lifecycle test (body testCrud: true)')
+    .option('--record-id <id>', 'Record ID for test (body recordId)')
+    .option('--no-cleanup', 'Disable cleanup after test (body cleanup: false)')
+    .option('--primary-key-value <value|@path>', 'Primary key value or path to JSON file (e.g. @pk.json) for body primaryKeyValue')
+    .option('--no-async', 'Use sync mode (no polling); single POST, no asyncRun')
+    .action(async(datasourceKey, options) => {
+      try {
+        const data = await runDatasourceTestE2E(datasourceKey, {
+          app: options.app,
+          environment: options.env,
+          debug: options.debug,
+          verbose: options.verbose,
+          async: options.async !== false,
+          testCrud: options.testCrud,
+          recordId: options.recordId,
+          cleanup: options.cleanup,
+          primaryKeyValue: options.primaryKeyValue
+        });
+        displayE2EResults(data, options.verbose);
+        const steps = data.steps || data.completedActions || [];
+        const failed = data.success === false || steps.some(s => s.success === false || s.error);
+        if (failed) process.exit(1);
+      } catch (error) {
+        logger.error(chalk.red('❌ E2E test failed:'), error.message);
         process.exit(1);
       }
     });
@@ -84,7 +153,9 @@ function setupDatasourceCommands(program) {
   setupDatasourceValidateCommand(datasource);
   setupDatasourceListCommand(datasource);
   setupDatasourceDiffCommand(datasource);
-  setupDatasourceDeployCommand(datasource);
+  setupDatasourceUploadCommand(datasource);
+  setupDatasourceTestIntegrationCommand(datasource);
+  setupDatasourceTestE2ECommand(datasource);
 }
 
 module.exports = { setupDatasourceCommands };

package/lib/commands/dev-init.js

@@ -14,6 +14,44 @@ const { generateCSR, getCertDir, readClientCertPem, readClientKeyPem, getCertVal
 const { getOrCreatePublicKeyContent } = require('../utils/ssh-key-helper');
 const devApi = require('../api/dev.api');
 const logger = require('../utils/logger');
+const {
+  isSslUntrustedError,
+  fetchInstallCa,
+  installCaPlatform,
+  promptInstallCa
+} = require('../utils/dev-ca-install');
+
+/**
+ * Ensure the Builder Server is trusted: run health check; on SSL untrusted error,
+ * optionally fetch and install CA, then retry.
+ * @param {string} baseUrl - Builder Server base URL
+ * @param {Object} options - Commander options (yes, y, no-install-ca)
+ * @returns {Promise<void>}
+ */
+async function ensureServerTrusted(baseUrl, options) {
+  const skipInstall = options['no-install-ca'];
+  const autoInstall = options.yes || options.y;
+  try {
+    await devApi.getHealth(baseUrl);
+  } catch (err) {
+    if (!isSslUntrustedError(err)) throw err;
+    const manualUrl = `${baseUrl.replace(/\/+$/, '')}/install-ca`;
+    if (skipInstall) {
+      throw new Error(`Server certificate not trusted. Install CA manually: ${manualUrl}`);
+    }
+    if (!autoInstall) {
+      const install = await promptInstallCa();
+      if (!install) {
+        throw new Error(`Server certificate not trusted. Install CA manually: ${manualUrl}`);
+      }
+    }
+    logger.log(chalk.gray('  Downloading and installing CA...'));
+    const caPem = await fetchInstallCa(baseUrl);
+    await installCaPlatform(caPem, baseUrl);
+    logger.log(chalk.gray('  CA installed. Retrying...'));
+    await devApi.getHealth(baseUrl);
+  }
+}
 
 /**
  * Validate init options and return normalized baseUrl and devId.
@@ -213,7 +251,7 @@ async function runDevInit(options) {
   logger.log(chalk.blue('\n🔐 Onboarding with Builder Server...\n'));
 
   try {
-    await devApi.getHealth(baseUrl);
+    await ensureServerTrusted(baseUrl, options);
   } catch (err) {
     throw new Error(`Cannot reach Builder Server at ${baseUrl}. Check URL and network. ${err.message}`);
   }

package/lib/commands/repair-auth-config.js

@@ -0,0 +1,99 @@
+/**
+ * Normalizes system file authentication.security and configuration keyvault entries.
+ * @fileoverview Repair auth/config KV_* names and path-style kv:// values
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+'use strict';
+
+const { systemKeyToKvPrefix, securityKeyToVar, kvEnvKeyToPath } = require('../utils/credential-secrets-env');
+
+/**
+ * Returns true if a kv value looks like legacy format (KeyVault suffix or no path segments).
+ * @param {string} val - Value from authentication.security or configuration
+ * @returns {boolean}
+ */
+function isLegacyKvValue(val) {
+  if (typeof val !== 'string' || !val.trim().toLowerCase().startsWith('kv://')) return false;
+  const after = val.trim().slice(5); // after 'kv://'
+  return after.includes('KeyVault') || !after.includes('/');
+}
+
+/**
+ * Normalizes authentication.security keyvault entries to path-style kv:// values (kv://systemKey/variable).
+ * @param {Object} security - authentication.security object (mutated)
+ * @param {string} prefix - KV prefix (e.g. 'HUBSPOT')
+ * @param {string} systemKey - System key for path namespace
+ * @param {string[]} changes - Array to append change descriptions to
+ * @returns {boolean} True if any change was made
+ */
+function normalizeSecuritySection(security, prefix, systemKey, changes) {
+  let updated = false;
+  for (const key of Object.keys(security)) {
+    const val = security[key];
+    if (typeof val !== 'string' || !isLegacyKvValue(val)) continue;
+    const envName = `KV_${prefix}_${securityKeyToVar(key)}`;
+    const pathVal = kvEnvKeyToPath(envName, systemKey);
+    if (pathVal) {
+      security[key] = pathVal;
+      changes.push(`authentication.security.${key}: normalized to path-style ${pathVal}`);
+      updated = true;
+    }
+  }
+  return updated;
+}
+
+/**
+ * Normalizes configuration array keyvault entries to canonical KV_* names and path-style values.
+ * @param {Object[]} config - configuration array (mutated)
+ * @param {string} prefix - KV prefix (e.g. 'HUBSPOT')
+ * @param {string} systemKey - System key for path namespace
+ * @param {string[]} changes - Array to append change descriptions to
+ * @returns {boolean} True if any change was made
+ */
+function normalizeConfigurationSection(config, prefix, systemKey, changes) {
+  let updated = false;
+  for (let i = 0; i < config.length; i++) {
+    const entry = config[i];
+    if (!entry || !entry.name || (entry.location !== 'keyvault' && !String(entry.name).startsWith('KV_'))) continue;
+    const afterPrefix = entry.name.startsWith(`KV_${prefix}_`)
+      ? entry.name.slice(`KV_${prefix}_`.length)
+      : entry.name.replace(/^KV_[A-Z0-9]+_/, '');
+    const normalizedVar = afterPrefix.replace(/_/g, '').toUpperCase();
+    const canonicalName = `KV_${prefix}_${normalizedVar}`;
+    const pathVal = kvEnvKeyToPath(canonicalName, systemKey);
+    if (!pathVal) continue;
+    const pathValWithoutPrefix = pathVal.replace(/^kv:\/\//, '');
+    const valueLegacy = typeof entry.value === 'string' && (entry.value.includes('KeyVault') || !entry.value.includes('/'));
+    if (entry.name !== canonicalName || (valueLegacy && entry.value !== pathValWithoutPrefix)) {
+      config[i] = { ...entry, name: canonicalName, value: pathValWithoutPrefix, location: 'keyvault' };
+      changes.push(`configuration: normalized ${entry.name} → ${canonicalName}, value → path-style`);
+      updated = true;
+    }
+  }
+  return updated;
+}
+
+/**
+ * Normalizes system file authentication.security and configuration keyvault entries to canonical
+ * KV_* names and path-style kv:// values so upload validation and env.template align.
+ *
+ * @param {Object} systemParsed - Parsed system config (mutated)
+ * @param {string} systemKey - System key (e.g. 'hubspot')
+ * @param {string[]} changes - Array to append change descriptions to
+ * @returns {boolean} True if any change was made
+ */
+function normalizeSystemFileAuthAndConfig(systemParsed, systemKey, changes) {
+  const prefix = systemKeyToKvPrefix(systemKey);
+  if (!prefix) return false;
+  const security = systemParsed.authentication?.security;
+  let updated = (security && typeof security === 'object' && normalizeSecuritySection(security, prefix, systemKey, changes));
+  const config = systemParsed.configuration;
+  if (Array.isArray(config)) {
+    updated = normalizeConfigurationSection(config, prefix, systemKey, changes) || updated;
+  }
+  return updated;
+}
+
+module.exports = { normalizeSystemFileAuthAndConfig, isLegacyKvValue };

package/lib/commands/repair-datasource-keys.js

@@ -0,0 +1,208 @@
+/**
+ * Normalize datasource keys and filenames to canonical form during repair.
+ *
+ * Key: <systemKey>-<resourceType> or <systemKey>-<resourceType>-2, -3 for duplicates.
+ * Filename: <systemKey>-datasource-<suffix>.<ext> where suffix = key without leading systemKey-.
+ * Skips keys/filenames that already match the valid pattern (e.g. customer-extra, customer-1).
+ *
+ * @fileoverview Datasource key and filename normalization for repair
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+const { loadConfigFile, writeConfigFile } = require('../utils/config-format');
+
+/**
+ * Returns suffix from a canonical-format filename: <systemKey>-datasource-<suffix>.<ext>.
+ *
+ * @param {string} fileName - Filename
+ * @param {string} systemKey - System key
+ * @returns {string|null} Suffix or null if not in canonical format
+ */
+function suffixFromCanonicalFilename(fileName, systemKey) {
+  const base = path.basename(fileName);
+  const ext = path.extname(fileName);
+  const withoutExt = base.slice(0, -ext.length);
+  const prefix = `${systemKey}-datasource-`;
+  if (!withoutExt.startsWith(prefix)) return null;
+  return withoutExt.slice(prefix.length) || null;
+}
+
+/**
+ * Returns true if the key already matches canonical form and should not be changed.
+ * Valid: <systemKey>-<resourceType> or <systemKey>-<resourceType>-<extra> (e.g. customer-extra, customer-1).
+ * When fileName is provided and is canonical, key may be just the suffix (e.g. record-storage).
+ * Invalid (will normalize): key ending with redundant -datasource (e.g. hubspot-demo-companies-datasource).
+ *
+ * @param {string} key - Datasource key
+ * @param {string} systemKey - System key
+ * @param {string} [fileName] - Optional filename; if canonical, key can be suffix-only
+ * @returns {boolean}
+ */
+function isKeyAlreadyCanonical(key, systemKey, fileName) {
+  if (!key || !systemKey) return false;
+  if (fileName && isFilenameAlreadyCanonical(fileName, systemKey)) {
+    const suffixFromFile = suffixFromCanonicalFilename(fileName, systemKey);
+    if (suffixFromFile && (key === suffixFromFile || key === `${systemKey}-${suffixFromFile}`)) {
+      return true;
+    }
+  }
+  if (!key.startsWith(systemKey + '-')) return false;
+  const suffix = key.slice(systemKey.length + 1);
+  if (!suffix) return false;
+  if (suffix.endsWith('-datasource')) return false;
+  return true;
+}
+
+/**
+ * Derives resourceType slug from key: strip systemKey prefix, then strip trailing -datasource if present.
+ *
+ * @param {string} key - Current datasource key
+ * @param {string} systemKey - System key
+ * @returns {string}
+ */
+function slugFromKey(key, systemKey) {
+  if (!key || !systemKey || !key.startsWith(systemKey + '-')) return key || '';
+  let suffix = key.slice(systemKey.length + 1);
+  if (suffix.endsWith('-datasource')) suffix = suffix.slice(0, -'-datasource'.length);
+  return suffix || key;
+}
+
+/**
+ * Returns canonical filename for a datasource: <systemKey>-datasource-<suffix>.<ext>.
+ *
+ * @param {string} canonicalKey - Canonical datasource key
+ * @param {string} systemKey - System key
+ * @param {string} ext - File extension including dot (e.g. .json)
+ * @returns {string}
+ */
+function canonicalDatasourceFilename(canonicalKey, systemKey, ext) {
+  const suffix = canonicalKey.startsWith(systemKey + '-')
+    ? canonicalKey.slice(systemKey.length + 1)
+    : canonicalKey;
+  return `${systemKey}-datasource-${suffix}${ext}`;
+}
+
+/**
+ * Returns true if filename already matches canonical pattern <systemKey>-datasource-<suffix>.<ext>.
+ *
+ * @param {string} fileName - Current filename
+ * @param {string} systemKey - System key
+ * @returns {boolean}
+ */
+function isFilenameAlreadyCanonical(fileName, systemKey) {
+  const base = path.basename(fileName);
+  const ext = path.extname(fileName);
+  const withoutExt = base.slice(0, -ext.length);
+  const prefix = `${systemKey}-datasource-`;
+  if (!withoutExt.startsWith(prefix)) return false;
+  const suffix = withoutExt.slice(prefix.length);
+  if (!suffix || suffix.endsWith('-datasource')) return false;
+  return true;
+}
+
+/**
+ * Normalizes datasource keys and filenames to canonical form. Runs early in repair.
+ * Updates file contents (key property), renames files when needed, and updates variables.externalIntegration.dataSources.
+ *
+ * @param {string} appPath - Application directory path
+ * @param {string[]} datasourceFiles - Current list of datasource filenames
+ * @param {string} systemKey - System key
+ * @param {Object} variables - Application variables (mutated: externalIntegration.dataSources updated)
+ * @param {boolean} dryRun - If true, do not write or rename
+ * @param {string[]} changes - Array to append change descriptions to
+ * @returns {{ updated: boolean, datasourceFiles: string[] }} Updated flag and new list of datasource filenames
+ */
+/* eslint-disable max-lines-per-function, max-statements, complexity -- Normalization loops and branching per file */
+function normalizeDatasourceKeysAndFilenames(appPath, datasourceFiles, systemKey, variables, dryRun, changes) {
+  if (!datasourceFiles || datasourceFiles.length === 0) {
+    return { updated: false, datasourceFiles: datasourceFiles || [] };
+  }
+
+  const slugCounts = new Map();
+  const fileInfos = [];
+
+  for (const fileName of datasourceFiles) {
+    const filePath = path.join(appPath, fileName);
+    if (!fs.existsSync(filePath)) continue;
+    let parsed;
+    try {
+      parsed = loadConfigFile(filePath);
+    } catch (err) {
+      fileInfos.push({ fileName, key: null, skip: true });
+      continue;
+    }
+    const key = (parsed && typeof parsed.key === 'string' && parsed.key.trim()) ? parsed.key.trim() : null;
+    if (isKeyAlreadyCanonical(key, systemKey, fileName) && isFilenameAlreadyCanonical(fileName, systemKey)) {
+      fileInfos.push({ fileName, key, skip: true });
+      continue;
+    }
+    const slug = slugFromKey(key || fileName, systemKey);
+    fileInfos.push({
+      fileName,
+      parsed,
+      key,
+      slug,
+      canonicalKey: null,
+      skip: false
+    });
+  }
+
+  for (const info of fileInfos) {
+    if (info.skip) continue;
+    const slug = info.slug;
+    const n = (slugCounts.get(slug) || 0) + 1;
+    slugCounts.set(slug, n);
+    info.canonicalKey = n === 1 ? `${systemKey}-${slug}` : `${systemKey}-${slug}-${n}`;
+  }
+
+  let updated = false;
+  const newDatasourceFiles = [];
+
+  for (const info of fileInfos) {
+    if (info.skip) {
+      newDatasourceFiles.push(info.fileName);
+      continue;
+    }
+    const { fileName, parsed, canonicalKey } = info;
+    const ext = path.extname(fileName);
+    const canonicalFileName = canonicalDatasourceFilename(canonicalKey, systemKey, ext);
+
+    if (parsed.key !== canonicalKey) {
+      parsed.key = canonicalKey;
+      if (!dryRun) writeConfigFile(path.join(appPath, fileName), parsed);
+      changes.push(`${fileName}: key → ${canonicalKey}`);
+      updated = true;
+    }
+    if (fileName !== canonicalFileName) {
+      const oldPath = path.join(appPath, fileName);
+      const newPath = path.join(appPath, canonicalFileName);
+      if (!dryRun && fs.existsSync(oldPath) && !fs.existsSync(newPath)) {
+        fs.renameSync(oldPath, newPath);
+      }
+      changes.push(`Renamed ${fileName} → ${canonicalFileName}`);
+      updated = true;
+      newDatasourceFiles.push(canonicalFileName);
+    } else {
+      newDatasourceFiles.push(fileName);
+    }
+  }
+
+  if (updated && variables.externalIntegration && Array.isArray(variables.externalIntegration.dataSources)) {
+    variables.externalIntegration.dataSources = [...newDatasourceFiles].sort();
+  }
+
+  return { updated, datasourceFiles: newDatasourceFiles };
+}
+
+module.exports = {
+  normalizeDatasourceKeysAndFilenames,
+  isKeyAlreadyCanonical,
+  slugFromKey,
+  canonicalDatasourceFilename,
+  isFilenameAlreadyCanonical
+};