@aifabrix/builder 2.6.2 → 2.7.0

package/lib/utils/schema-loader.js

@@ -0,0 +1,220 @@
+/**
+ * Schema Loading Utilities
+ *
+ * Loads and compiles JSON schemas for validation.
+ * Provides schema type detection and cached validators.
+ *
+ * @fileoverview Schema loading utilities for AI Fabrix Builder
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fs = require('fs');
+const path = require('path');
+const Ajv = require('ajv');
+
+// Cache for compiled validators
+// These are reset when module is reloaded (for testing)
+let externalSystemValidator = null;
+let externalDataSourceValidator = null;
+
+/**
+ * Reset validators cache (for testing)
+ * @function resetValidators
+ */
+function resetValidators() {
+  externalSystemValidator = null;
+  externalDataSourceValidator = null;
+}
+
+/**
+ * Loads and compiles external-system schema
+ * Caches the compiled validator for performance
+ *
+ * @function loadExternalSystemSchema
+ * @returns {Function} Compiled AJV validator function
+ * @throws {Error} If schema file cannot be loaded or compiled
+ *
+ * @example
+ * const validate = loadExternalSystemSchema();
+ * const valid = validate(data);
+ */
+function loadExternalSystemSchema() {
+  if (externalSystemValidator) {
+    return externalSystemValidator;
+  }
+
+  const schemaPath = path.join(__dirname, '..', 'schema', 'external-system.schema.json');
+
+  if (!fs.existsSync(schemaPath)) {
+    throw new Error(`External system schema not found: ${schemaPath}`);
+  }
+
+  const schemaContent = fs.readFileSync(schemaPath, 'utf8');
+  let schema;
+
+  try {
+    schema = JSON.parse(schemaContent);
+  } catch (error) {
+    throw new Error(`Invalid JSON in external-system.schema.json: ${error.message}`);
+  }
+
+  const ajv = new Ajv({ allErrors: true, strict: false });
+  externalSystemValidator = ajv.compile(schema);
+
+  return externalSystemValidator;
+}
+
+/**
+ * Loads and compiles external-datasource schema
+ * Caches the compiled validator for performance
+ *
+ * @function loadExternalDataSourceSchema
+ * @returns {Function} Compiled AJV validator function
+ * @throws {Error} If schema file cannot be loaded or compiled
+ *
+ * @example
+ * const validate = loadExternalDataSourceSchema();
+ * const valid = validate(data);
+ */
+function loadExternalDataSourceSchema() {
+  if (externalDataSourceValidator) {
+    return externalDataSourceValidator;
+  }
+
+  const schemaPath = path.join(__dirname, '..', 'schema', 'external-datasource.schema.json');
+
+  if (!fs.existsSync(schemaPath)) {
+    throw new Error(`External datasource schema not found: ${schemaPath}`);
+  }
+
+  const schemaContent = fs.readFileSync(schemaPath, 'utf8');
+  let schema;
+
+  try {
+    schema = JSON.parse(schemaContent);
+  } catch (error) {
+    throw new Error(`Invalid JSON in external-datasource.schema.json: ${error.message}`);
+  }
+
+  // For draft-2020-12 schemas, we need to handle $schema differently
+  // Remove $schema if it's draft-2020-12 to avoid AJV issues
+  const schemaToCompile = { ...schema };
+  if (schemaToCompile.$schema && schemaToCompile.$schema.includes('2020-12')) {
+    // AJV v8 supports draft-2020-12 but may need the schema without $schema for compilation
+    delete schemaToCompile.$schema;
+  }
+
+  const ajv = new Ajv({ allErrors: true, strict: false, strictSchema: false });
+  externalDataSourceValidator = ajv.compile(schemaToCompile);
+
+  return externalDataSourceValidator;
+}
+
+/**
+ * Detects schema type from file content or path
+ * Attempts to identify if file is application, external-system, or external-datasource
+ *
+ * @function detectSchemaType
+ * @param {string} filePath - Path to the file
+ * @param {string} [content] - Optional file content (if not provided, will be read from file)
+ * @returns {string} Schema type: 'application' | 'external-system' | 'external-datasource'
+ * @throws {Error} If file cannot be read or parsed
+ *
+ * @example
+ * const type = detectSchemaType('./hubspot.json');
+ * // Returns: 'external-system'
+ */
+function detectSchemaType(filePath, content) {
+  let fileContent = content;
+
+  // Read file if content not provided
+  if (!fileContent) {
+    if (!fs.existsSync(filePath)) {
+      throw new Error(`File not found: ${filePath}`);
+    }
+    fileContent = fs.readFileSync(filePath, 'utf8');
+  }
+
+  // Try to parse JSON
+  let parsed;
+  try {
+    parsed = JSON.parse(fileContent);
+  } catch (error) {
+    throw new Error(`Invalid JSON in file: ${error.message}`);
+  }
+
+  // Check for schema type indicators
+  // Check $id for schema type
+  if (parsed.$id) {
+    if (parsed.$id.includes('external-system')) {
+      return 'external-system';
+    }
+    if (parsed.$id.includes('external-datasource')) {
+      return 'external-datasource';
+    }
+    if (parsed.$id.includes('application-schema')) {
+      return 'application';
+    }
+  }
+
+  // Check title for schema type (works even without $id or $schema)
+  if (parsed.title) {
+    const titleLower = parsed.title.toLowerCase();
+    if (titleLower.includes('external system') || titleLower.includes('external-system') || titleLower.includes('external system configuration')) {
+      return 'external-system';
+    }
+    if (titleLower.includes('external data source') || titleLower.includes('external datasource') || titleLower.includes('external-datasource')) {
+      return 'external-datasource';
+    }
+    if (titleLower.includes('application')) {
+      return 'application';
+    }
+  }
+
+  // Check for required fields to determine type
+  // External system requires: key, displayName, description, type, authentication
+  if (parsed.key && parsed.displayName && parsed.type && parsed.authentication) {
+    // Check if it has systemKey (datasource) or not (system)
+    if (parsed.systemKey) {
+      return 'external-datasource';
+    }
+    // Check if type is one of external-system types
+    if (['openapi', 'mcp', 'custom'].includes(parsed.type)) {
+      return 'external-system';
+    }
+  }
+
+  // Check for datasource-specific fields
+  if (parsed.systemKey && parsed.entityKey && parsed.fieldMappings) {
+    return 'external-datasource';
+  }
+
+  // Check for application-specific fields
+  if (parsed.deploymentKey || (parsed.image && parsed.registryMode && parsed.port)) {
+    return 'application';
+  }
+
+  // Fallback: check filename pattern
+  const fileName = path.basename(filePath).toLowerCase();
+  if (fileName.includes('external-system') || fileName.includes('external_system')) {
+    return 'external-system';
+  }
+  if (fileName.includes('external-datasource') || fileName.includes('external_datasource') || fileName.includes('datasource')) {
+    return 'external-datasource';
+  }
+  if (fileName.includes('application') || fileName.includes('variables') || fileName.includes('deploy')) {
+    return 'application';
+  }
+
+  // Default to application if cannot determine
+  return 'application';
+}
+
+module.exports = {
+  loadExternalSystemSchema,
+  loadExternalDataSourceSchema,
+  detectSchemaType,
+  resetValidators
+};
+

package/lib/utils/schema-resolver.js

@@ -0,0 +1,174 @@
+/**
+ * Schema Resolution Utilities
+ *
+ * Resolves paths for external integration schemas from application configuration.
+ * Handles schemaBasePath resolution and external file discovery.
+ *
+ * @fileoverview Schema path resolution utilities for AI Fabrix Builder
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+/**
+ * Resolves schemaBasePath from application variables.yaml
+ * Supports both absolute and relative paths
+ *
+ * @async
+ * @function resolveSchemaBasePath
+ * @param {string} appName - Application name
+ * @returns {Promise<string>} Resolved absolute path to schema base directory
+ * @throws {Error} If variables.yaml not found, externalIntegration missing, or path invalid
+ *
+ * @example
+ * const basePath = await resolveSchemaBasePath('myapp');
+ * // Returns: '/path/to/builder/myapp/schemas'
+ */
+async function resolveSchemaBasePath(appName) {
+  if (!appName || typeof appName !== 'string') {
+    throw new Error('App name is required and must be a string');
+  }
+
+  const variablesPath = path.join(process.cwd(), 'builder', appName, 'variables.yaml');
+
+  if (!fs.existsSync(variablesPath)) {
+    throw new Error(`variables.yaml not found: ${variablesPath}`);
+  }
+
+  const content = fs.readFileSync(variablesPath, 'utf8');
+  let variables;
+
+  try {
+    variables = yaml.load(content);
+  } catch (error) {
+    throw new Error(`Invalid YAML syntax in variables.yaml: ${error.message}`);
+  }
+
+  // Check if externalIntegration block exists
+  if (!variables.externalIntegration) {
+    throw new Error(`externalIntegration block not found in variables.yaml for app: ${appName}`);
+  }
+
+  if (!variables.externalIntegration.schemaBasePath) {
+    throw new Error(`schemaBasePath not found in externalIntegration block for app: ${appName}`);
+  }
+
+  const schemaBasePath = variables.externalIntegration.schemaBasePath;
+  const variablesDir = path.dirname(variablesPath);
+
+  // Resolve path (absolute or relative to variables.yaml location)
+  let resolvedPath;
+  if (path.isAbsolute(schemaBasePath)) {
+    resolvedPath = schemaBasePath;
+  } else {
+    resolvedPath = path.resolve(variablesDir, schemaBasePath);
+  }
+
+  // Normalize path
+  resolvedPath = path.normalize(resolvedPath);
+
+  // Validate path exists
+  if (!fs.existsSync(resolvedPath)) {
+    throw new Error(`Schema base path does not exist: ${resolvedPath}`);
+  }
+
+  if (!fs.statSync(resolvedPath).isDirectory()) {
+    throw new Error(`Schema base path is not a directory: ${resolvedPath}`);
+  }
+
+  return resolvedPath;
+}
+
+/**
+ * Resolves all external system and datasource files from application configuration
+ * Returns array of file paths with metadata
+ *
+ * @async
+ * @function resolveExternalFiles
+ * @param {string} appName - Application name
+ * @returns {Promise<Array<{path: string, type: 'system'|'datasource', fileName: string}>>} Array of resolved file paths with metadata
+ * @throws {Error} If files cannot be resolved or do not exist
+ *
+ * @example
+ * const files = await resolveExternalFiles('myapp');
+ * // Returns: [
+ * //   { path: '/path/to/hubspot.json', type: 'system', fileName: 'hubspot.json' },
+ * //   { path: '/path/to/hubspot-deal.json', type: 'datasource', fileName: 'hubspot-deal.json' }
+ * // ]
+ */
+async function resolveExternalFiles(appName) {
+  if (!appName || typeof appName !== 'string') {
+    throw new Error('App name is required and must be a string');
+  }
+
+  const variablesPath = path.join(process.cwd(), 'builder', appName, 'variables.yaml');
+
+  if (!fs.existsSync(variablesPath)) {
+    throw new Error(`variables.yaml not found: ${variablesPath}`);
+  }
+
+  const content = fs.readFileSync(variablesPath, 'utf8');
+  let variables;
+
+  try {
+    variables = yaml.load(content);
+  } catch (error) {
+    throw new Error(`Invalid YAML syntax in variables.yaml: ${error.message}`);
+  }
+
+  // Check if externalIntegration block exists
+  if (!variables.externalIntegration) {
+    return []; // No external integration, return empty array
+  }
+
+  // Resolve schema base path
+  const schemaBasePath = await resolveSchemaBasePath(appName);
+  const resolvedFiles = [];
+
+  // Resolve systems files
+  if (variables.externalIntegration.systems && Array.isArray(variables.externalIntegration.systems)) {
+    for (const systemFile of variables.externalIntegration.systems) {
+      const systemPath = path.join(schemaBasePath, systemFile);
+      const normalizedPath = path.normalize(systemPath);
+
+      if (!fs.existsSync(normalizedPath)) {
+        throw new Error(`External system file not found: ${normalizedPath}`);
+      }
+
+      resolvedFiles.push({
+        path: normalizedPath,
+        type: 'system',
+        fileName: systemFile
+      });
+    }
+  }
+
+  // Resolve datasources files
+  if (variables.externalIntegration.dataSources && Array.isArray(variables.externalIntegration.dataSources)) {
+    for (const datasourceFile of variables.externalIntegration.dataSources) {
+      const datasourcePath = path.join(schemaBasePath, datasourceFile);
+      const normalizedPath = path.normalize(datasourcePath);
+
+      if (!fs.existsSync(normalizedPath)) {
+        throw new Error(`External datasource file not found: ${normalizedPath}`);
+      }
+
+      resolvedFiles.push({
+        path: normalizedPath,
+        type: 'datasource',
+        fileName: datasourceFile
+      });
+    }
+  }
+
+  return resolvedFiles;
+}
+
+module.exports = {
+  resolveSchemaBasePath,
+  resolveExternalFiles
+};
+

package/lib/utils/secrets-helpers.js

@@ -13,10 +13,10 @@ const path = require('path');
 const yaml = require('js-yaml');
 const config = require('../config');
 const { buildHostnameToServiceMap, resolveUrlPort } = require('./secrets-utils');
-const { rewriteInfraEndpoints, getEnvHosts } = require('./env-endpoints');
+const { rewriteInfraEndpoints, getEnvHosts, getServicePort, getServiceHost, getLocalhostOverride } = require('./env-endpoints');
 const { loadEnvConfig } = require('./env-config-loader');
-const { processEnvVariables } = require('./env-copy');
 const { updateContainerPortInEnvFile } = require('./env-ports');
+const { buildEnvVarMap } = require('./env-map');
 
 /**
  * Interpolate ${VAR} occurrences with values from envVars map
@@ -262,6 +262,24 @@ async function adjustLocalEnvPortsInContent(envContent, variablesPath) {
   // Update infra endpoints with developer-id adjusted ports for local context
   updated = await rewriteInfraEndpoints(updated, 'local');
 
+  // Interpolate ${VAR} references created by rewriteInfraEndpoints
+  // Get the ports that were just set by rewriteInfraEndpoints for interpolation
+  const hostsForPorts = await getEnvHosts('local');
+  const redisPort = await getServicePort('REDIS_PORT', 'redis', hostsForPorts, 'local');
+  const dbPort = await getServicePort('DB_PORT', 'postgres', hostsForPorts, 'local');
+  const localhostOverride = getLocalhostOverride('local');
+  const redisHost = getServiceHost(hostsForPorts.REDIS_HOST, 'local', 'localhost', localhostOverride);
+  const dbHost = getServiceHost(hostsForPorts.DB_HOST, 'local', 'localhost', localhostOverride);
+
+  // Build envVars map and ensure it has the correct values
+  const envVars = await buildEnvVarMap('local', null, devIdNum);
+  // Override with the actual values that were just set by rewriteInfraEndpoints
+  envVars.REDIS_HOST = redisHost;
+  envVars.REDIS_PORT = String(redisPort);
+  envVars.DB_HOST = dbHost;
+  envVars.DB_PORT = String(dbPort);
+  updated = interpolateEnvVars(updated, envVars);
+
   return updated;
 }
 
@@ -276,6 +294,29 @@ function readYamlAtPath(filePath) {
   return yaml.load(content);
 }
 
+/**
+ * Merge a single secret value from canonical into result
+ * @function mergeSecretValue
+ * @param {Object} result - Result object to merge into
+ * @param {string} key - Secret key
+ * @param {*} canonicalValue - Value from canonical secrets
+ */
+function mergeSecretValue(result, key, canonicalValue) {
+  const currentValue = result[key];
+  // Fill missing, empty, or undefined values
+  if (!(key in result) || currentValue === undefined || currentValue === null || currentValue === '') {
+    result[key] = canonicalValue;
+    return;
+  }
+  // Only replace values that are encrypted (have secure:// prefix)
+  // Plaintext values (no secure://) are used as-is
+  if (typeof currentValue === 'string' && typeof canonicalValue === 'string') {
+    if (currentValue.startsWith('secure://')) {
+      result[key] = canonicalValue;
+    }
+  }
+}
+
 /**
  * Apply canonical secrets path override if configured and file exists
  * @async
@@ -287,21 +328,29 @@ async function applyCanonicalSecretsOverride(currentSecrets) {
   let mergedSecrets = currentSecrets || {};
   try {
     const canonicalPath = await config.getSecretsPath();
-    if (canonicalPath) {
-      const resolvedCanonical = path.isAbsolute(canonicalPath)
-        ? canonicalPath
-        : path.resolve(process.cwd(), canonicalPath);
-      if (fs.existsSync(resolvedCanonical)) {
-        const configSecrets = readYamlAtPath(resolvedCanonical);
-        // Apply canonical secrets as a fallback source:
-        // - Do NOT override any existing keys from user/build
-        // - Add only missing keys from canonical path
-        if (configSecrets && typeof configSecrets === 'object') {
-          const result = { ...configSecrets, ...mergedSecrets };
-          mergedSecrets = result;
-        }
-      }
+    if (!canonicalPath) {
+      return mergedSecrets;
+    }
+    const resolvedCanonical = path.isAbsolute(canonicalPath)
+      ? canonicalPath
+      : path.resolve(process.cwd(), canonicalPath);
+    if (!fs.existsSync(resolvedCanonical)) {
+      return mergedSecrets;
+    }
+    const configSecrets = readYamlAtPath(resolvedCanonical);
+    if (!configSecrets || typeof configSecrets !== 'object') {
+      return mergedSecrets;
+    }
+    // Apply canonical secrets as a fallback source:
+    // - Do NOT override any existing keys from user/build
+    // - Add only missing keys from canonical path
+    // - Also fill in empty/undefined values from canonical path
+    // - Replace encrypted values (secure://) with canonical plaintext
+    const result = { ...mergedSecrets };
+    for (const [key, canonicalValue] of Object.entries(configSecrets)) {
+      mergeSecretValue(result, key, canonicalValue);
     }
+    mergedSecrets = result;
   } catch {
     // ignore and fall through
   }
@@ -351,7 +400,6 @@ module.exports = {
   replaceKvInContent,
   resolveServicePortsInEnvContent,
   loadEnvTemplate,
-  processEnvVariables,
   updateContainerPortInEnvFile,
   adjustLocalEnvPortsInContent,
   readYamlAtPath,

package/lib/utils/token-manager.js

@@ -86,6 +86,17 @@ function isTokenExpired(expiresAt) {
   return config.isTokenExpired(expiresAt);
 }
 
+/**
+ * Check if token should be refreshed proactively
+ * Returns true if token is within 15 minutes of expiry
+ * This helps keep Keycloak sessions alive by refreshing before the SSO Session Idle timeout (30 minutes)
+ * @param {string} expiresAt - ISO timestamp string
+ * @returns {boolean} True if token should be refreshed proactively
+ */
+function shouldRefreshToken(expiresAt) {
+  return config.shouldRefreshToken(expiresAt);
+}
+
 /**
  * Refresh client token using credentials from secrets.local.yaml
  * Gets new token from API and saves it to config.yaml
@@ -184,7 +195,7 @@ async function getOrRefreshClientToken(environment, appName, controllerUrl) {
  * @param {string} controllerUrl - Controller URL
  * @param {string} refreshToken - Refresh token
  * @returns {Promise<{token: string, refreshToken: string, expiresAt: string}>} New token info
- * @throws {Error} If refresh fails
+ * @throws {Error} If refresh fails or refresh token is expired/invalid
  */
 async function refreshDeviceToken(controllerUrl, refreshToken) {
   if (!controllerUrl || typeof controllerUrl !== 'string') {
@@ -194,27 +205,41 @@ async function refreshDeviceToken(controllerUrl, refreshToken) {
     throw new Error('Refresh token is required');
   }
 
-  // Call API refresh endpoint
-  const tokenResponse = await apiRefreshDeviceToken(controllerUrl, refreshToken);
+  try {
+    // Call API refresh endpoint
+    const tokenResponse = await apiRefreshDeviceToken(controllerUrl, refreshToken);
 
-  const token = tokenResponse.access_token;
-  const newRefreshToken = tokenResponse.refresh_token || refreshToken; // Use new refresh token if provided, otherwise keep old one
-  const expiresIn = tokenResponse.expires_in || 3600;
-  const expiresAt = new Date(Date.now() + expiresIn * 1000).toISOString();
+    const token = tokenResponse.access_token;
+    const newRefreshToken = tokenResponse.refresh_token || refreshToken; // Use new refresh token if provided, otherwise keep old one
+    const expiresIn = tokenResponse.expires_in || 3600;
+    const expiresAt = new Date(Date.now() + expiresIn * 1000).toISOString();
 
-  // Save new token and refresh token to config
-  await config.saveDeviceToken(controllerUrl, token, newRefreshToken, expiresAt);
+    // Save new token and refresh token to config
+    await config.saveDeviceToken(controllerUrl, token, newRefreshToken, expiresAt);
 
-  return {
-    token,
-    refreshToken: newRefreshToken,
-    expiresAt
-  };
+    return {
+      token,
+      refreshToken: newRefreshToken,
+      expiresAt
+    };
+  } catch (error) {
+    // Check if error indicates refresh token expiry (case-insensitive)
+    const errorMessage = (error.message || String(error)).toLowerCase();
+    if (errorMessage.includes('expired') ||
+        errorMessage.includes('invalid') ||
+        errorMessage.includes('401') ||
+        errorMessage.includes('unauthorized')) {
+      throw new Error('Refresh token has expired. Please login again using: aifabrix login');
+    }
+    // Re-throw other errors as-is
+    throw error;
+  }
 }
 
 /**
  * Get or refresh device token for controller
- * Checks if token exists and is valid, refreshes if expired using refresh token
+ * Checks if token exists and is valid, refreshes proactively if within 15 minutes of expiry
+ * This helps keep Keycloak sessions alive by refreshing before the SSO Session Idle timeout (30 minutes)
  * @param {string} controllerUrl - Controller URL
  * @returns {Promise<{token: string, controller: string}|null>} Token and controller URL, or null if not available
  */
@@ -226,18 +251,23 @@ async function getOrRefreshDeviceToken(controllerUrl) {
     return null;
   }
 
-  // Check if token is expired
-  if (!isTokenExpired(tokenInfo.expiresAt)) {
-    // Token is valid
+  // Check if token should be refreshed proactively (within 15 minutes of expiry)
+  // This ensures we refresh before Keycloak's SSO Session Idle timeout (30 minutes)
+  const needsRefresh = shouldRefreshToken(tokenInfo.expiresAt);
+
+  if (!needsRefresh) {
+    // Token is valid and doesn't need refresh yet
     return {
       token: tokenInfo.token,
       controller: tokenInfo.controller
     };
   }
 
-  // Token is expired, try to refresh if refresh token exists
+  // Token needs refresh (expired or within 15 minutes of expiry)
+  // Try to refresh if refresh token exists
   if (!tokenInfo.refreshToken) {
     // No refresh token available
+    logger.warn('Access token expired and no refresh token available. Please login again using: aifabrix login');
     return null;
   }
 
@@ -248,8 +278,13 @@ async function getOrRefreshDeviceToken(controllerUrl) {
       controller: controllerUrl
     };
   } catch (error) {
-    // Refresh failed, return null
-    logger.warn(`Failed to refresh device token: ${error.message}`);
+    // Refresh failed - check if it's a refresh token expiry
+    const errorMessage = error.message || String(error);
+    if (errorMessage.includes('Refresh token has expired')) {
+      logger.warn(`Refresh token expired: ${errorMessage}`);
+    } else {
+      logger.warn(`Failed to refresh device token: ${errorMessage}`);
+    }
     return null;
   }
 }
@@ -370,6 +405,7 @@ module.exports = {
   getDeviceToken,
   getClientToken,
   isTokenExpired,
+  shouldRefreshToken,
   refreshClientToken,
   refreshDeviceToken,
   loadClientCredentials,