@aifabrix/builder 2.41.0 → 2.42.1

package/lib/utils/token-manager.js

@@ -247,48 +247,29 @@ async function tryClientTokenAuth(environment, appName, controllerUrl) {
     const clientToken = await getOrRefreshClientToken(environment, appName, controllerUrl);
     if (clientToken && clientToken.token) {
       return {
-        type: 'bearer',
+        type: 'client-token',
         token: clientToken.token,
         controller: clientToken.controller
       };
     }
   } catch {
-    // Client token unavailable; getDeploymentAuth will try client credentials next (no warning here to avoid misleading output when env credentials succeed)
-  }
-  return null;
-}
-
-/**
- * Tries to get client credentials for deployment auth
- * @async
- * @function tryClientCredentialsAuth
- * @param {string} appName - Application name
- * @param {string} controllerUrl - Controller URL
- * @returns {Promise<Object|null>} Auth config with client credentials or null
- */
-async function tryClientCredentialsAuth(appName, controllerUrl) {
-  const credentials = await loadClientCredentials(appName);
-  if (credentials && credentials.clientId && credentials.clientSecret) {
-    return {
-      type: 'client-credentials',
-      clientId: credentials.clientId,
-      clientSecret: credentials.clientSecret,
-      controller: controllerUrl
-    };
+    // Client token unavailable; getDeploymentAuth will try exchanging credentials for token (no warning here to avoid misleading output when refresh succeeds)
   }
   return null;
 }
 
 /**
  * Get deployment authentication configuration with priority:
- * 1. Device token (Bearer) - for user-level audit tracking (preferred)
- * 2. Client token (Bearer) - for application-level authentication
- * 3. Client credentials (x-client-id/x-client-secret) - direct credential authentication
+ * 1. Device token → type 'bearer' (user token) → send as Authorization: Bearer
+ * 2. Client token → type 'client-token' (application token) → send as x-client-token header
+ * 3. When no token available: if client credentials exist, exchange for client token and return type 'client-token'.
+ *
+ * x-client-id/x-client-secret are used only at the token-issuing endpoint (e.g. POST /api/v1/auth/token).
  *
  * @param {string} controllerUrl - Controller URL
  * @param {string} environment - Environment key
  * @param {string} appName - Application name
- * @returns {Promise<{type: 'bearer'|'client-credentials', token?: string, clientId?: string, clientSecret?: string, controller: string}>} Auth configuration
+ * @returns {Promise<{type: 'bearer'|'client-token', token: string, controller: string}>} Auth config: bearer = user token, client-token = app token (x-client-token header)
  * @throws {Error} If no authentication method is available
  */
 async function getDeploymentAuth(controllerUrl, environment, appName) {
@@ -306,10 +287,21 @@ async function getDeploymentAuth(controllerUrl, environment, appName) {
     return clientTokenAuth;
   }
 
-  // Priority 3: Use client credentials directly
-  const credentialsAuth = await tryClientCredentialsAuth(appName, controllerUrl);
-  if (credentialsAuth) {
-    return credentialsAuth;
+  // Priority 3: Exchange client credentials for a token (never return client-credentials for app endpoints)
+  const credentials = await loadClientCredentials(appName);
+  if (credentials && credentials.clientId && credentials.clientSecret) {
+    try {
+      const refreshed = await refreshClientToken(environment, appName, controllerUrl);
+      if (refreshed && refreshed.token) {
+        return {
+          type: 'client-token',
+          token: refreshed.token,
+          controller: controllerUrl
+        };
+      }
+    } catch {
+      // Refresh failed; fall through to throw below
+    }
   }
 
   throw new Error(`No authentication method available. Run 'aifabrix login' for device token, or add credentials to ~/.aifabrix/secrets.local.yaml as '${appName}-client-idKeyVault' and '${appName}-client-secretKeyVault'`);
@@ -337,7 +329,7 @@ async function extractClientCredentials(authConfig, appKey, envKey, _options = {
     };
   }
 
-  if (authConfig.type === 'bearer') {
+  if (authConfig.type === 'bearer' || authConfig.type === 'client-token') {
     if (authConfig.clientId && authConfig.clientSecret) {
       return {
         clientId: authConfig.clientId,

package/lib/validation/env-template-auth.js

@@ -0,0 +1,157 @@
+/**
+ * env.template authentication kv:// validation helpers
+ *
+ * Collects required kv paths from system configs and extracts kv paths from env.template
+ * for validating that external integrations have all authentication secrets in env.template.
+ *
+ * @fileoverview Auth kv validation for env.template
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const { loadExternalIntegrationConfig, loadSystemFile } = require('../generator/external');
+
+/**
+ * Extracts all kv:// paths from env.template content (RHS of VAR=value lines).
+ * Uses same regex as validateKvReferencesInLines.
+ *
+ * @function extractKvPathsFromEnvTemplate
+ * @param {string} content - env.template file content
+ * @returns {Set<string>} Set of kv:// paths found (e.g. kv://hubspot/client-id)
+ *
+ * @example
+ * const paths = extractKvPathsFromEnvTemplate('CLIENT_ID=kv://hubspot/client-id\nPORT=3000');
+ * // paths has 'kv://hubspot/client-id'
+ */
+function extractKvPathsFromEnvTemplate(content) {
+  const paths = new Set();
+  const lines = content.split('\n');
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    if (!trimmed.includes('=')) continue;
+    const [_key, value] = trimmed.split('=', 2);
+    const val = (value || '').trim();
+    const matches = val.match(/kv:\/\/[^\s]*/g) || [];
+    for (const fullRef of matches) {
+      paths.add(fullRef);
+    }
+  }
+  return paths;
+}
+
+/**
+ * Extracts kv:// paths from commented lines in env.template (e.g. # KEY=kv://path or # kv://path).
+ * Used to treat commented-out keys as intentionally disabled for auth-coverage validation.
+ * Scans the whole line after '#' so both key=value and bare/commented refs are recognized.
+ *
+ * @function extractKvPathsFromCommentedLines
+ * @param {string} content - env.template file content
+ * @returns {Set<string>} Set of kv:// paths found in commented lines (e.g. kv://avoma/apikey)
+ */
+function extractKvPathsFromCommentedLines(content) {
+  const paths = new Set();
+  const lines = content.split('\n');
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed.startsWith('#')) continue;
+    const afterHash = trimmed.slice(1).trim();
+    const matches = afterHash.match(/kv:\/\/[^\s]*/g) || [];
+    for (const fullRef of matches) {
+      paths.add(fullRef);
+    }
+  }
+  return paths;
+}
+
+/**
+ * Returns true if the required path is present in the set or matches any entry case-insensitively.
+ * Used so commented-out keys match required paths regardless of kv path casing (e.g. apiKey vs apikey).
+ *
+ * @function setHasPathIgnoreCase
+ * @param {Set<string>} pathSet - Set of kv paths (e.g. from commented lines)
+ * @param {string} requiredPath - Required path from authentication.security
+ * @returns {boolean} True if requiredPath is in pathSet or matches any element when lowercased
+ */
+function setHasPathIgnoreCase(pathSet, requiredPath) {
+  if (pathSet.has(requiredPath)) return true;
+  const requiredLower = requiredPath.toLowerCase();
+  for (const p of pathSet) {
+    if (p.toLowerCase() === requiredLower) return true;
+  }
+  return false;
+}
+
+/**
+ * Collects required kv:// paths from authentication.security of all system configs.
+ * For external integrations only. On config load failure, returns empty set and optional warning.
+ *
+ * @async
+ * @function collectRequiredAuthKvPaths
+ * @param {string} appPath - Application directory path
+ * @param {Object} [options] - Options (reserved)
+ * @returns {Promise<{ requiredPaths: Set<string>, warning?: string }>} Required kv paths and optional warning
+ *
+ * @example
+ * const { requiredPaths } = await collectRequiredAuthKvPaths('/path/to/integration/hubspot');
+ * // requiredPaths has kv:// paths from authentication.security
+ */
+async function collectRequiredAuthKvPaths(appPath, _options = {}) {
+  const requiredPaths = new Set();
+  try {
+    const { schemaBasePath, systemFiles } = await loadExternalIntegrationConfig(appPath);
+    for (const systemFileName of systemFiles) {
+      const systemJson = await loadSystemFile(appPath, schemaBasePath, systemFileName);
+      const security = systemJson.authentication?.security;
+      if (!security || typeof security !== 'object') continue;
+      for (const val of Object.values(security)) {
+        if (typeof val === 'string' && /^kv:\/\/.+/.test(val)) {
+          requiredPaths.add(val);
+        }
+      }
+    }
+    return { requiredPaths };
+  } catch (error) {
+    return {
+      requiredPaths: new Set(),
+      warning: `Could not validate auth kv coverage (skip auth check): ${error.message}`
+    };
+  }
+}
+
+/**
+ * Validates that env.template covers all authentication.security kv paths for external apps.
+ * Modifies errors and warnings in place.
+ *
+ * @async
+ * @function validateAuthKvCoverage
+ * @param {string} appPath - Application path
+ * @param {string} content - env.template content
+ * @param {string[]} errors - Errors array to push to
+ * @param {string[]} warnings - Warnings array to push to
+ * @param {Object} [options] - Options
+ */
+async function validateAuthKvCoverage(appPath, content, errors, warnings, options = {}) {
+  const authResult = await collectRequiredAuthKvPaths(appPath, options);
+  if (authResult.warning) warnings.push(authResult.warning);
+  if (authResult.requiredPaths.size === 0) return;
+  const actualPaths = extractKvPathsFromEnvTemplate(content);
+  const commentedPaths = extractKvPathsFromCommentedLines(content);
+  for (const requiredPath of authResult.requiredPaths) {
+    const inActive = actualPaths.has(requiredPath);
+    const inCommented = setHasPathIgnoreCase(commentedPaths, requiredPath);
+    if (!inActive && !inCommented) {
+      errors.push(
+        `env.template: Missing required authentication secret (required by authentication.security): add a variable with value ${requiredPath}`
+      );
+    }
+  }
+}
+
+module.exports = {
+  extractKvPathsFromEnvTemplate,
+  extractKvPathsFromCommentedLines,
+  setHasPathIgnoreCase,
+  collectRequiredAuthKvPaths,
+  validateAuthKvCoverage
+};

package/lib/validation/env-template-kv.js

@@ -0,0 +1,41 @@
+/**
+ * env.template kv:// reference validation (syntax only).
+ * Skips comment and empty lines. Used by validator.validateEnvTemplate.
+ *
+ * @fileoverview Kv reference validation for env.template lines
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+/**
+ * Validates kv:// references in env.template lines; pushes errors into the given array.
+ * Skips empty and comment (#) lines.
+ *
+ * @function validateKvReferencesInLines
+ * @param {string[]} lines - Lines of env.template content
+ * @param {string[]} errors - Array to push error messages into
+ */
+function validateKvReferencesInLines(lines, errors) {
+  lines.forEach((line, index) => {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) {
+      return;
+    }
+    const matches = line.match(/kv:\/\/[^\s]*/g) || [];
+    for (const fullRef of matches) {
+      const pathMatch = fullRef.match(/^kv:\/\/(.*)$/);
+      const pathStr = pathMatch ? pathMatch[1] : '';
+      const invalid = !pathStr || pathStr.startsWith('/') || pathStr.endsWith('/');
+      if (invalid) {
+        const hint = !pathStr
+          ? 'path is empty (use kv://secret-key)'
+          : pathStr.startsWith('/')
+            ? 'path must not start with / (use kv://secret-key not kv:///secret-key)'
+            : 'path must not end with / (use kv://secret-key not kv://secret-key/)';
+        errors.push(`env.template line ${index + 1}: Invalid kv:// reference "${fullRef}" - ${hint}`);
+      }
+    }
+  });
+}
+
+module.exports = { validateKvReferencesInLines };

package/lib/validation/external-manifest-validator.js

@@ -156,6 +156,30 @@ function validateRequiredFields(manifest, errors) {
   });
 }
 
+/**
+ * Validates that each datasource's systemKey matches the application system key.
+ * Matches dataplane upload API wording so integrators recognize the error.
+ *
+ * @function validateDatasourceSystemKeyAlignment
+ * @param {Object} manifest - Manifest object with system and dataSources
+ * @param {Array} errors - Errors array to append to
+ * @returns {void}
+ */
+function validateDatasourceSystemKeyAlignment(manifest, errors) {
+  const systemKey = manifest.system?.key;
+  if (!manifest.dataSources || !Array.isArray(manifest.dataSources) || systemKey === null || systemKey === undefined || systemKey === '') {
+    return;
+  }
+  manifest.dataSources.forEach((datasource) => {
+    if (datasource.systemKey !== systemKey) {
+      const dsKey = datasource.key || 'unknown';
+      errors.push(
+        `Data source '${dsKey}' systemKey does not match application system key (expected '${systemKey}', got '${datasource.systemKey}')`
+      );
+    }
+  });
+}
+
 /**
  * Validates controller deployment manifest for external systems
  * Validates manifest structure and inline system/dataSources against their schemas
@@ -187,6 +211,7 @@ async function validateControllerManifest(manifest) {
   validateManifestStructure(manifest, ajv, applicationSchema, errors);
   validateInlineSystem(manifest, ajv, externalSystemSchema, errors);
   validateDatasources(manifest, ajv, externalDatasourceSchema, errors, warnings);
+  validateDatasourceSystemKeyAlignment(manifest, errors);
   validateConditionalRequirements(manifest, errors, warnings);
   validateRequiredFields(manifest, errors);
 

package/lib/validation/external-system-auth-rules.js

@@ -0,0 +1,86 @@
+/**
+ * External system authentication rules (OAuth2/AAD grantType, authorizationUrl, configuration).
+ * Used after schema validation for external system files.
+ *
+ * @fileoverview OAuth2/AAD and configuration rules for external system configs
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const VALID_GRANT_TYPES = ['client_credentials', 'authorization_code'];
+
+/** Standard auth variable names (credential parameters supplied at runtime). Not allowed in configuration except BASEURL when auth is none. */
+const STANDARD_AUTH_VAR_NAMES = new Set([
+  'baseurl', 'clientid', 'clientsecret', 'tokenurl', 'apikey', 'username', 'password'
+]);
+
+function trimVar(value) {
+  return (value !== undefined && value !== null ? String(value).trim() : '');
+}
+
+function isOAuth2OrAad(method) {
+  const m = (method && String(method).toLowerCase()) || '';
+  return m === 'oauth2' || m === 'aad';
+}
+
+/**
+ * Validates OAuth2/AAD grantType and conditional authorizationUrl for external system files.
+ * When method is oauth2 or aad: grantType (if present) must be client_credentials or authorization_code;
+ * when effective grant is authorization_code (explicit or default), authorizationUrl is required.
+ *
+ * @function validateOAuth2GrantTypeAndAuthorizationUrl
+ * @param {Object} parsed - Parsed external system object (must have authentication.variables when method is oauth2/aad)
+ * @param {string[]} errors - Array to push validation error messages into
+ */
+function validateOAuth2GrantTypeAndAuthorizationUrl(parsed, errors) {
+  const auth = parsed?.authentication;
+  const variables = auth?.variables;
+  if (!variables || typeof variables !== 'object' || !isOAuth2OrAad(auth.method)) {
+    return;
+  }
+
+  const grantType = trimVar(variables.grantType);
+  if (grantType !== '' && !VALID_GRANT_TYPES.includes(grantType)) {
+    errors.push('authentication.variables.grantType must be one of: client_credentials, authorization_code');
+    return;
+  }
+
+  const effectiveGrant = grantType || 'authorization_code';
+  if (effectiveGrant === 'authorization_code' && trimVar(variables.authorizationUrl) === '') {
+    errors.push('authentication.variables.authorizationUrl is required when grantType is authorization_code or omitted');
+  }
+}
+
+/**
+ * Validates that the external system configuration array does not contain standard auth variable names.
+ * BASEURL, CLIENTID, CLIENTSECRET, TOKENURL, APIKEY, USERNAME, PASSWORD are credential parameters
+ * supplied from the selected credential at runtime and must not be in configuration. Exception:
+ * BASEURL is only allowed in configuration when authentication.method is 'none'.
+ *
+ * @param {Object} parsed - Parsed external system object
+ * @param {string[]} errors - Array to push validation error messages into
+ */
+function validateConfigurationNoStandardAuthVariables(parsed, errors) {
+  const config = parsed?.configuration;
+  if (!Array.isArray(config) || config.length === 0) return;
+  const method = (parsed?.authentication?.method && String(parsed.authentication.method).toLowerCase()) || '';
+  const authNone = method === 'none';
+  const allowedWhenNone = new Set(['baseurl']);
+  for (const item of config) {
+    const name = (item?.name && String(item.name).trim()) || '';
+    if (!name) continue;
+    const nameLower = name.toLowerCase();
+    if (!STANDARD_AUTH_VAR_NAMES.has(nameLower)) continue;
+    if (authNone && allowedWhenNone.has(nameLower)) continue;
+    errors.push(
+      `configuration must not contain standard auth variable '${name}'. ` +
+      'Standard auth variables (BASEURL, CLIENTID, CLIENTSECRET, TOKENURL, APIKEY, USERNAME, PASSWORD) are supplied from the selected credential at runtime. ' +
+      'BASEURL is only allowed in configuration when authentication.method is \'none\'.'
+    );
+  }
+}
+
+module.exports = {
+  validateOAuth2GrantTypeAndAuthorizationUrl,
+  validateConfigurationNoStandardAuthVariables
+};

package/lib/validation/validate-batch.js

@@ -0,0 +1,149 @@
+/**
+ * Batch validation helpers for validate --integration / --builder.
+ * @fileoverview Builds batch results and runs validation across multiple apps
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const { listIntegrationAppNames, listBuilderAppNames } = require('../utils/paths');
+
+/**
+ * Collects error strings from a single validation result (various shapes).
+ * @param {Object} result - Single-app validation result
+ * @returns {string[]} Error messages
+ */
+function collectResultErrors(result) {
+  const errs = [];
+  if (result.errors && Array.isArray(result.errors)) {
+    errs.push(...result.errors);
+  }
+  if (result.application && result.application.errors && Array.isArray(result.application.errors)) {
+    errs.push(...result.application.errors);
+  }
+  if (result.steps) {
+    ['application', 'components', 'manifest'].forEach(step => {
+      const s = result.steps[step];
+      if (s && s.errors && Array.isArray(s.errors)) {
+        errs.push(...s.errors);
+      }
+    });
+  }
+  return errs;
+}
+
+/**
+ * Collects warning strings from a single validation result.
+ * @param {Object} result - Single-app validation result
+ * @returns {string[]} Warning messages
+ */
+function collectResultWarnings(result) {
+  const w = [];
+  if (result.warnings && Array.isArray(result.warnings)) {
+    w.push(...result.warnings);
+  }
+  if (result.application && result.application.warnings && Array.isArray(result.application.warnings)) {
+    w.push(...result.application.warnings);
+  }
+  if (result.steps) {
+    ['application', 'components', 'manifest'].forEach(step => {
+      const s = result.steps[step];
+      if (s && s.warnings && Array.isArray(s.warnings)) {
+        w.push(...s.warnings);
+      }
+    });
+  }
+  return w;
+}
+
+/**
+ * Builds batch result from per-app results. Each item has appName and either result or error.
+ * @param {Array<{appName: string, result?: Object, error?: string}>} items - Per-app results
+ * @returns {{ batch: true, valid: boolean, results: Array, errors: string[], warnings: string[] }}
+ */
+function buildBatchResult(items) {
+  const errors = [];
+  const warnings = [];
+  items.forEach(item => {
+    if (item.error) {
+      errors.push(`${item.appName}: ${item.error}`);
+    } else if (item.result) {
+      collectResultErrors(item.result).forEach(e => errors.push(`${item.appName}: ${e}`));
+      collectResultWarnings(item.result).forEach(w => warnings.push(`${item.appName}: ${w}`));
+    }
+  });
+  const valid = items.every(item => item.result && item.result.valid);
+  return {
+    batch: true,
+    valid,
+    results: items,
+    errors,
+    warnings
+  };
+}
+
+/**
+ * Validates all apps under integration/ (each as external system).
+ * @async
+ * @param {Function} validateAppOrFile - validateAppOrFile(appName, options) from validate.js
+ * @param {Object} [options] - Validation options
+ * @returns {Promise<{ batch: true, valid: boolean, results: Array, errors: string[], warnings: string[] }>}
+ */
+async function validateAllIntegrations(validateAppOrFile, options = {}) {
+  const names = listIntegrationAppNames();
+  const items = [];
+  for (const appName of names) {
+    try {
+      const result = await validateAppOrFile(appName, options);
+      items.push({ appName, result });
+    } catch (error) {
+      items.push({ appName, error: error.message || String(error) });
+    }
+  }
+  return buildBatchResult(items);
+}
+
+/**
+ * Validates all apps under builder/.
+ * @async
+ * @param {Function} validateAppOrFile - validateAppOrFile(appName, options) from validate.js
+ * @param {Object} [options] - Validation options
+ * @returns {Promise<{ batch: true, valid: boolean, results: Array, errors: string[], warnings: string[] }>}
+ */
+async function validateAllBuilderApps(validateAppOrFile, options = {}) {
+  const names = listBuilderAppNames();
+  const items = [];
+  for (const appName of names) {
+    try {
+      const result = await validateAppOrFile(appName, options);
+      items.push({ appName, result });
+    } catch (error) {
+      items.push({ appName, error: error.message || String(error) });
+    }
+  }
+  return buildBatchResult(items);
+}
+
+/**
+ * Validates all integration and builder apps in one run.
+ * @async
+ * @param {Function} validateAppOrFile - validateAppOrFile(appName, options) from validate.js
+ * @param {Object} [options] - Validation options
+ * @returns {Promise<{ batch: true, valid: boolean, results: Array, errors: string[], warnings: string[] }>}
+ */
+async function validateAll(validateAppOrFile, options = {}) {
+  const [integrationResult, builderResult] = await Promise.all([
+    validateAllIntegrations(validateAppOrFile, options),
+    validateAllBuilderApps(validateAppOrFile, options)
+  ]);
+  const mergedResults = [...integrationResult.results, ...builderResult.results];
+  return buildBatchResult(mergedResults);
+}
+
+module.exports = {
+  buildBatchResult,
+  collectResultErrors,
+  collectResultWarnings,
+  validateAllIntegrations,
+  validateAllBuilderApps,
+  validateAll
+};

package/lib/validation/validate-datasource-keys-api.js

@@ -0,0 +1,33 @@
+/**
+ * @fileoverview Async validation of datasourceKeys against dataplane API
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const { getPlatformDetails } = require('../api/wizard.api');
+const { validateDatasourceKeysForPlatform } = require('./wizard-datasource-validation');
+
+/**
+ * Validate datasourceKeys against platform's available datasources; throws if invalid
+ * @async
+ * @param {string} dataplaneUrl - Dataplane URL
+ * @param {Object} authConfig - Auth config
+ * @param {string} platformKey - Platform key
+ * @param {string[]} datasourceKeys - Datasource keys to validate
+ * @throws {Error} If any key is invalid
+ */
+async function validateDatasourceKeysBeforePlatformConfig(dataplaneUrl, authConfig, platformKey, datasourceKeys) {
+  if (!Array.isArray(datasourceKeys) || datasourceKeys.length === 0) return;
+  const platformDetails = await getPlatformDetails(dataplaneUrl, authConfig, platformKey);
+  const datasources = platformDetails?.data?.datasources ?? platformDetails?.datasources ?? [];
+  const { valid, invalidKeys } = validateDatasourceKeysForPlatform(datasourceKeys, datasources);
+  if (!valid) {
+    const availableKeys = datasources.map(d => d.key).filter(Boolean);
+    throw new Error(
+      `Invalid datasource keys: [${invalidKeys.join(', ')}]. ` +
+      `Available for platform '${platformKey}': [${availableKeys.join(', ')}].`
+    );
+  }
+}
+
+module.exports = { validateDatasourceKeysBeforePlatformConfig };