@aifabrix/builder 2.6.3 → 2.7.0

package/lib/schema/external-system.schema.json

@@ -0,0 +1,262 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://aifabrix.ai/schemas/external-system.schema.json",
+  "title": "AI Fabrix External System Configuration Schema",
+  "description": "Schema for configuring an external system connected to the AI Fabrix Dataplane. This defines authentication, OpenAPI/MCP bindings, field mappings defaults, metadata handling and portal inputs.",
+
+  "metadata": {
+    "key": "external-system-schema",
+    "name": "External System Configuration Schema",
+    "description": "JSON schema for validating ExternalSystem configuration files",
+    "version": "1.0.0",
+    "type": "schema",
+    "category": "integration",
+    "author": "AI Fabrix Team",
+    "createdAt": "2024-01-01T00:00:00Z",
+    "updatedAt": "2024-01-01T00:00:00Z",
+    "compatibility": {
+      "minVersion": "1.0.0",
+      "maxVersion": "2.0.0",
+      "deprecated": false
+    },
+    "tags": ["schema", "external-system", "dataplane", "integration", "validation"],
+    "dependencies": [],
+    "changelog": [
+      {
+        "version": "1.0.0",
+        "date": "2024-01-01T00:00:00Z",
+        "changes": [
+          "Initial schema for External Systems",
+          "Added OpenAPI and MCP contract configuration",
+          "Added authentication and environment variables",
+          "Added portalInput for UI-driven onboarding",
+          "Compatible with field-mappings.schema and security-model.schema"
+        ],
+        "breaking": false
+      }
+    ]
+  },
+
+  "type": "object",
+
+  "required": [
+    "key",
+    "displayName",
+    "description",
+    "type",
+    "authentication"
+  ],
+
+  "properties": {
+    "key": {
+      "type": "string",
+      "description": "Unique external system identifier (cannot be changed after creation). Example: 'hubspot', 'salesforce', 'teams'.",
+      "pattern": "^[a-z0-9-]+$",
+      "minLength": 3,
+      "maxLength": 40
+    },
+
+    "displayName": {
+      "type": "string",
+      "description": "Human-readable name for the external system",
+      "minLength": 1,
+      "maxLength": 100
+    },
+
+    "description": {
+      "type": "string",
+      "description": "Description of this external system integration",
+      "minLength": 1,
+      "maxLength": 500
+    },
+
+    "type": {
+      "type": "string",
+      "enum": ["openapi", "mcp", "custom"],
+      "description": "Integration type: OpenAPI-driven, MCP-driven, or custom Python connector."
+    },
+
+    "enabled": {
+      "type": "boolean",
+      "default": true
+    },
+
+    "environment": {
+      "type": "object",
+      "description": "Environment-level configuration values used by dataplane and external data sources.",
+      "properties": {
+        "baseUrl": {
+          "type": "string",
+          "description": "Base API URL or MCP server URL",
+          "pattern": "^(http|https)://.*$"
+        },
+        "region": {
+          "type": "string",
+          "description": "Optional region setting for API routing"
+        }
+      },
+      "additionalProperties": true
+    },
+
+    "authentication": {
+      "type": "object",
+      "description": "Authentication configuration for the external system.",
+      "required": ["mode"],
+      "properties": {
+        "mode": {
+          "type": "string",
+          "enum": ["oauth2", "apikey", "basic", "aad", "none"],
+          "description": "Authentication method used for API access."
+        },
+
+        "oauth2": {
+          "type": "object",
+          "description": "OAuth2 configuration used when mode == 'oauth2'. All secrets are stored in Key Vault.",
+          "properties": {
+            "tokenUrl": {
+              "type": "string",
+              "pattern": "^(http|https)://.*$"
+            },
+            "clientId": {
+              "type": "string"
+            },
+            "clientSecret": {
+              "type": "string"
+            },
+            "scopes": {
+              "type": "array",
+              "items": { "type": "string" }
+            }
+          },
+          "additionalProperties": false
+        },
+
+        "apikey": {
+          "type": "object",
+          "properties": {
+            "headerName": { "type": "string" },
+            "key": { "type": "string" }
+          },
+          "additionalProperties": false
+        },
+
+        "basic": {
+          "type": "object",
+          "properties": {
+            "username": { "type": "string" },
+            "password": { "type": "string" }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "openapi": {
+      "type": "object",
+      "description": "OpenAPI integration configuration.",
+      "properties": {
+        "documentKey": {
+          "type": "string",
+          "description": "Reference to OpenAPI spec registered via builder. Example: 'hubspot-v3'."
+        },
+        "autoDiscoverEntities": {
+          "type": "boolean",
+          "default": false,
+          "description": "Automatically discover resources/entities from OpenAPI schema."
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "mcp": {
+      "type": "object",
+      "description": "Model Context Protocol integration config.",
+      "properties": {
+        "serverUrl": {
+          "type": "string",
+          "pattern": "^(http|https)://.*$"
+        },
+        "toolPrefix": {
+          "type": "string",
+          "pattern": "^[a-z0-9-]+$",
+          "description": "Prefix for MCP tool names generated from this system."
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "dataSources": {
+      "type": "array",
+      "description": "List of data source keys belonging to this external system. Each must match external-datasource.schema.json.",
+      "items": {
+        "type": "string",
+        "pattern": "^[a-z0-9-]+$"
+      },
+      "uniqueItems": true
+    },
+
+    "configuration": {
+      "type": "array",
+      "description": "External system configuration variables (same pattern as application-schema).",
+      "items": {
+        "type": "object",
+        "required": ["name", "value", "location", "required"],
+        "properties": {
+          "name": {
+            "type": "string",
+            "pattern": "^[A-Z_][A-Z0-9_]*$"
+          },
+          "value": {
+            "type": "string",
+            "description": "Literal value or parameter reference ({{parameter}})"
+          },
+          "location": {
+            "type": "string",
+            "enum": ["variable", "keyvault"]
+          },
+          "required": {
+            "type": "boolean"
+          },
+          "portalInput": {
+            "type": "object",
+            "required": ["field", "label"],
+            "properties": {
+              "field": {
+                "type": "string",
+                "enum": ["password", "text", "textarea", "select", "json"]
+              },
+              "label": { "type": "string" },
+              "placeholder": { "type": "string" },
+              "options": {
+                "type": "array",
+                "items": { "type": "string" }
+              },
+              "masked": { "type": "boolean" },
+              "validation": {
+                "type": "object",
+                "properties": {
+                  "minLength": { "type": "integer" },
+                  "maxLength": { "type": "integer" },
+                  "pattern": { "type": "string" },
+                  "required": { "type": "boolean" }
+                },
+                "additionalProperties": false
+              }
+            },
+            "additionalProperties": false
+          }
+        },
+        "additionalProperties": false
+      }
+    },
+
+    "tags": {
+      "type": "array",
+      "description": "Optional tags for search / filtering",
+      "items": { "type": "string" }
+    }
+  },
+
+  "additionalProperties": false
+}

package/lib/secrets.js

@@ -20,7 +20,6 @@ const {
   formatMissingSecretsFileInfo,
   replaceKvInContent,
   loadEnvTemplate,
-  processEnvVariables,
   adjustLocalEnvPortsInContent,
   rewriteInfraEndpoints,
   readYamlAtPath,
@@ -28,6 +27,7 @@ const {
   ensureNonEmptySecrets,
   validateSecrets
 } = require('./utils/secrets-helpers');
+const { processEnvVariables } = require('./utils/env-copy');
 const { buildEnvVarMap } = require('./utils/env-map');
 const { resolveServicePortsInEnvContent } = require('./utils/secrets-url');
 const {
@@ -291,6 +291,25 @@ async function applyEnvironmentTransformations(resolved, environment, variablesP
   if (environment === 'docker') {
     resolved = await resolveServicePortsInEnvContent(resolved, environment);
     resolved = await rewriteInfraEndpoints(resolved, 'docker');
+    // Interpolate ${VAR} references created by rewriteInfraEndpoints
+    // Get the actual host and port values from env-endpoints.js directly
+    // to ensure they are correctly populated in envVars for interpolation
+    const { getEnvHosts, getServiceHost, getServicePort, getLocalhostOverride } = require('./utils/env-endpoints');
+    const hosts = await getEnvHosts('docker');
+    const localhostOverride = getLocalhostOverride('docker');
+    const redisHost = getServiceHost(hosts.REDIS_HOST, 'docker', 'redis', localhostOverride);
+    const redisPort = await getServicePort('REDIS_PORT', 'redis', hosts, 'docker', null);
+    const dbHost = getServiceHost(hosts.DB_HOST, 'docker', 'postgres', localhostOverride);
+    const dbPort = await getServicePort('DB_PORT', 'postgres', hosts, 'docker', null);
+
+    // Build envVars map and ensure it has the correct values
+    const envVars = await buildEnvVarMap('docker');
+    // Override with the actual values that were just set by rewriteInfraEndpoints
+    envVars.REDIS_HOST = redisHost;
+    envVars.REDIS_PORT = String(redisPort);
+    envVars.DB_HOST = dbHost;
+    envVars.DB_PORT = String(dbPort);
+    resolved = interpolateEnvVars(resolved, envVars);
     resolved = await updatePortForDocker(resolved, variablesPath);
   } else if (environment === 'local') {
     // adjustLocalEnvPortsInContent handles both PORT and infra endpoints

package/lib/utils/env-copy.js

@@ -16,6 +16,8 @@ const logger = require('./logger');
 const config = require('../config');
 const devConfig = require('../utils/dev-config');
 const { rewriteInfraEndpoints } = require('./env-endpoints');
+const { buildEnvVarMap } = require('./env-map');
+const { interpolateEnvVars } = require('./secrets-helpers');
 
 /**
  * Process and optionally copy env file to envOutputPath if configured
@@ -120,6 +122,28 @@ async function processEnvVariables(envPath, variablesPath, appName, secretsPath)
     });
     // Rewrite infra endpoints using env-config mapping for local context
     envContent = await rewriteInfraEndpoints(envContent, 'local', infraPorts);
+    // Interpolate ${VAR} references created by rewriteInfraEndpoints
+    // Extract actual values from updated content to use for interpolation
+    const envVars = await buildEnvVarMap('local', null, devIdNum);
+    // Extract REDIS_HOST, REDIS_PORT, DB_HOST, DB_PORT from updated content if present
+    // Only extract if the value doesn't contain ${VAR} references (to avoid circular interpolation)
+    const redisHostMatch = envContent.match(/^REDIS_HOST\s*=\s*([^\r\n$]+)/m);
+    const redisPortMatch = envContent.match(/^REDIS_PORT\s*=\s*([^\r\n$]+)/m);
+    const dbHostMatch = envContent.match(/^DB_HOST\s*=\s*([^\r\n$]+)/m);
+    const dbPortMatch = envContent.match(/^DB_PORT\s*=\s*([^\r\n$]+)/m);
+    if (redisHostMatch && redisHostMatch[1] && !redisHostMatch[1].includes('${')) {
+      envVars.REDIS_HOST = redisHostMatch[1].trim();
+    }
+    if (redisPortMatch && redisPortMatch[1] && !redisPortMatch[1].includes('${')) {
+      envVars.REDIS_PORT = redisPortMatch[1].trim();
+    }
+    if (dbHostMatch && dbHostMatch[1] && !dbHostMatch[1].includes('${')) {
+      envVars.DB_HOST = dbHostMatch[1].trim();
+    }
+    if (dbPortMatch && dbPortMatch[1] && !dbPortMatch[1].includes('${')) {
+      envVars.DB_PORT = dbPortMatch[1].trim();
+    }
+    envContent = interpolateEnvVars(envContent, envVars);
     fs.writeFileSync(outputPath, envContent, { mode: 0o600 });
     logger.log(chalk.green(`✓ Copied .env to: ${variables.build.envOutputPath}`));
   }

package/lib/utils/env-endpoints.js

@@ -149,27 +149,26 @@ function updateEndpointVariables(envContent, redisHost, redisPort, dbHost, dbPor
 
   // Update REDIS_URL if present
   if (/^REDIS_URL\s*=.*$/m.test(updated)) {
-    const m = updated.match(/^REDIS_URL\s*=\s*redis:\/\/([^:\s]+):\d+/m);
-    const currentHost = m && m[1] ? m[1] : null;
-    const targetHost = redisHost || currentHost;
-    if (targetHost) {
-      updated = updated.replace(
-        /^REDIS_URL\s*=\s*.*$/m,
-        `REDIS_URL=redis://${targetHost}:${redisPort}`
-      );
-    }
+    updated = updated.replace(
+      /^REDIS_URL\s*=\s*.*$/m,
+      'REDIS_URL=redis://${REDIS_HOST}:${REDIS_PORT}'
+    );
   }
 
   // Update REDIS_HOST if present
+  // If the original has host:port pattern, use ${VAR} references
+  // Otherwise, just set the host value
   if (/^REDIS_HOST\s*=.*$/m.test(updated)) {
     const hostPortMatch = updated.match(/^REDIS_HOST\s*=\s*([a-zA-Z0-9_.-]+):\d+$/m);
     const hasPortPattern = !!hostPortMatch;
     if (hasPortPattern) {
+      // Original had host:port pattern, use ${VAR} references
       updated = updated.replace(
         /^REDIS_HOST\s*=\s*.*$/m,
-        `REDIS_HOST=${redisHost}:${redisPort}`
+        'REDIS_HOST=${REDIS_HOST}:${REDIS_PORT}'
       );
     } else {
+      // Just host, set actual value
       updated = updated.replace(/^REDIS_HOST\s*=\s*.*$/m, `REDIS_HOST=${redisHost}`);
     }
   }
@@ -195,6 +194,31 @@ function updateEndpointVariables(envContent, redisHost, redisPort, dbHost, dbPor
     );
   }
 
+  // Update DATABASE_URL and other database URL variables (postgresql:// or postgres:// URLs)
+  // Handles DATABASE_URL, DATABASELOG_URL, and any other *_URL variables with postgresql:// or postgres://
+  // First, collect all matches to avoid issues with modifying string during iteration
+  const dbUrlPattern = /^(DATABASE[^=]*_URL)\s*=\s*(postgresql?:\/\/)([^:]+):([^@]+)@([^:/]+):(\d+)(\/[^\s]*)?/gm;
+  const matches = [];
+  let dbUrlMatch;
+  // Reset regex lastIndex to ensure we start from beginning
+  dbUrlPattern.lastIndex = 0;
+  while ((dbUrlMatch = dbUrlPattern.exec(updated)) !== null) {
+    matches.push({
+      varName: dbUrlMatch[1], // DATABASE_URL, DATABASELOG_URL, etc.
+      protocol: dbUrlMatch[2], // postgresql:// or postgres://
+      user: dbUrlMatch[3], // username
+      password: dbUrlMatch[4], // password
+      path: dbUrlMatch[7] || '' // /database path
+    });
+  }
+  // Now apply all replacements
+  for (const match of matches) {
+    updated = updated.replace(
+      new RegExp(`^${match.varName}\\s*=\\s*.*$`, 'm'),
+      `${match.varName}=${match.protocol}${match.user}:${match.password}@\${DB_HOST}:\${DB_PORT}${match.path}`
+    );
+  }
+
   // Update DATABASE_PORT if present (some templates use DATABASE_PORT instead of DB_PORT)
   if (/^DATABASE_PORT\s*=.*$/m.test(updated)) {
     updated = updated.replace(
@@ -203,6 +227,20 @@ function updateEndpointVariables(envContent, redisHost, redisPort, dbHost, dbPor
     );
   }
 
+  // Update Keycloak database variables if present
+  // KC_DB_URL_HOST is used by Keycloak to connect to the database
+  if (/^KC_DB_URL_HOST\s*=.*$/m.test(updated)) {
+    updated = updated.replace(/^KC_DB_URL_HOST\s*=\s*.*$/m, `KC_DB_URL_HOST=${dbHost}`);
+  }
+
+  // KC_DB_URL_PORT is used by Keycloak for database port
+  if (/^KC_DB_URL_PORT\s*=.*$/m.test(updated)) {
+    updated = updated.replace(
+      /^KC_DB_URL_PORT\s*=\s*.*$/m,
+      `KC_DB_URL_PORT=${dbPort}`
+    );
+  }
+
   return updated;
 }
 
@@ -259,6 +297,7 @@ module.exports = {
   splitHost,
   getServicePort,
   getServiceHost,
-  updateEndpointVariables
+  updateEndpointVariables,
+  getLocalhostOverride
 };
 

package/lib/utils/schema-loader.js

@@ -0,0 +1,220 @@
+/**
+ * Schema Loading Utilities
+ *
+ * Loads and compiles JSON schemas for validation.
+ * Provides schema type detection and cached validators.
+ *
+ * @fileoverview Schema loading utilities for AI Fabrix Builder
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fs = require('fs');
+const path = require('path');
+const Ajv = require('ajv');
+
+// Cache for compiled validators
+// These are reset when module is reloaded (for testing)
+let externalSystemValidator = null;
+let externalDataSourceValidator = null;
+
+/**
+ * Reset validators cache (for testing)
+ * @function resetValidators
+ */
+function resetValidators() {
+  externalSystemValidator = null;
+  externalDataSourceValidator = null;
+}
+
+/**
+ * Loads and compiles external-system schema
+ * Caches the compiled validator for performance
+ *
+ * @function loadExternalSystemSchema
+ * @returns {Function} Compiled AJV validator function
+ * @throws {Error} If schema file cannot be loaded or compiled
+ *
+ * @example
+ * const validate = loadExternalSystemSchema();
+ * const valid = validate(data);
+ */
+function loadExternalSystemSchema() {
+  if (externalSystemValidator) {
+    return externalSystemValidator;
+  }
+
+  const schemaPath = path.join(__dirname, '..', 'schema', 'external-system.schema.json');
+
+  if (!fs.existsSync(schemaPath)) {
+    throw new Error(`External system schema not found: ${schemaPath}`);
+  }
+
+  const schemaContent = fs.readFileSync(schemaPath, 'utf8');
+  let schema;
+
+  try {
+    schema = JSON.parse(schemaContent);
+  } catch (error) {
+    throw new Error(`Invalid JSON in external-system.schema.json: ${error.message}`);
+  }
+
+  const ajv = new Ajv({ allErrors: true, strict: false });
+  externalSystemValidator = ajv.compile(schema);
+
+  return externalSystemValidator;
+}
+
+/**
+ * Loads and compiles external-datasource schema
+ * Caches the compiled validator for performance
+ *
+ * @function loadExternalDataSourceSchema
+ * @returns {Function} Compiled AJV validator function
+ * @throws {Error} If schema file cannot be loaded or compiled
+ *
+ * @example
+ * const validate = loadExternalDataSourceSchema();
+ * const valid = validate(data);
+ */
+function loadExternalDataSourceSchema() {
+  if (externalDataSourceValidator) {
+    return externalDataSourceValidator;
+  }
+
+  const schemaPath = path.join(__dirname, '..', 'schema', 'external-datasource.schema.json');
+
+  if (!fs.existsSync(schemaPath)) {
+    throw new Error(`External datasource schema not found: ${schemaPath}`);
+  }
+
+  const schemaContent = fs.readFileSync(schemaPath, 'utf8');
+  let schema;
+
+  try {
+    schema = JSON.parse(schemaContent);
+  } catch (error) {
+    throw new Error(`Invalid JSON in external-datasource.schema.json: ${error.message}`);
+  }
+
+  // For draft-2020-12 schemas, we need to handle $schema differently
+  // Remove $schema if it's draft-2020-12 to avoid AJV issues
+  const schemaToCompile = { ...schema };
+  if (schemaToCompile.$schema && schemaToCompile.$schema.includes('2020-12')) {
+    // AJV v8 supports draft-2020-12 but may need the schema without $schema for compilation
+    delete schemaToCompile.$schema;
+  }
+
+  const ajv = new Ajv({ allErrors: true, strict: false, strictSchema: false });
+  externalDataSourceValidator = ajv.compile(schemaToCompile);
+
+  return externalDataSourceValidator;
+}
+
+/**
+ * Detects schema type from file content or path
+ * Attempts to identify if file is application, external-system, or external-datasource
+ *
+ * @function detectSchemaType
+ * @param {string} filePath - Path to the file
+ * @param {string} [content] - Optional file content (if not provided, will be read from file)
+ * @returns {string} Schema type: 'application' | 'external-system' | 'external-datasource'
+ * @throws {Error} If file cannot be read or parsed
+ *
+ * @example
+ * const type = detectSchemaType('./hubspot.json');
+ * // Returns: 'external-system'
+ */
+function detectSchemaType(filePath, content) {
+  let fileContent = content;
+
+  // Read file if content not provided
+  if (!fileContent) {
+    if (!fs.existsSync(filePath)) {
+      throw new Error(`File not found: ${filePath}`);
+    }
+    fileContent = fs.readFileSync(filePath, 'utf8');
+  }
+
+  // Try to parse JSON
+  let parsed;
+  try {
+    parsed = JSON.parse(fileContent);
+  } catch (error) {
+    throw new Error(`Invalid JSON in file: ${error.message}`);
+  }
+
+  // Check for schema type indicators
+  // Check $id for schema type
+  if (parsed.$id) {
+    if (parsed.$id.includes('external-system')) {
+      return 'external-system';
+    }
+    if (parsed.$id.includes('external-datasource')) {
+      return 'external-datasource';
+    }
+    if (parsed.$id.includes('application-schema')) {
+      return 'application';
+    }
+  }
+
+  // Check title for schema type (works even without $id or $schema)
+  if (parsed.title) {
+    const titleLower = parsed.title.toLowerCase();
+    if (titleLower.includes('external system') || titleLower.includes('external-system') || titleLower.includes('external system configuration')) {
+      return 'external-system';
+    }
+    if (titleLower.includes('external data source') || titleLower.includes('external datasource') || titleLower.includes('external-datasource')) {
+      return 'external-datasource';
+    }
+    if (titleLower.includes('application')) {
+      return 'application';
+    }
+  }
+
+  // Check for required fields to determine type
+  // External system requires: key, displayName, description, type, authentication
+  if (parsed.key && parsed.displayName && parsed.type && parsed.authentication) {
+    // Check if it has systemKey (datasource) or not (system)
+    if (parsed.systemKey) {
+      return 'external-datasource';
+    }
+    // Check if type is one of external-system types
+    if (['openapi', 'mcp', 'custom'].includes(parsed.type)) {
+      return 'external-system';
+    }
+  }
+
+  // Check for datasource-specific fields
+  if (parsed.systemKey && parsed.entityKey && parsed.fieldMappings) {
+    return 'external-datasource';
+  }
+
+  // Check for application-specific fields
+  if (parsed.deploymentKey || (parsed.image && parsed.registryMode && parsed.port)) {
+    return 'application';
+  }
+
+  // Fallback: check filename pattern
+  const fileName = path.basename(filePath).toLowerCase();
+  if (fileName.includes('external-system') || fileName.includes('external_system')) {
+    return 'external-system';
+  }
+  if (fileName.includes('external-datasource') || fileName.includes('external_datasource') || fileName.includes('datasource')) {
+    return 'external-datasource';
+  }
+  if (fileName.includes('application') || fileName.includes('variables') || fileName.includes('deploy')) {
+    return 'application';
+  }
+
+  // Default to application if cannot determine
+  return 'application';
+}
+
+module.exports = {
+  loadExternalSystemSchema,
+  loadExternalDataSourceSchema,
+  detectSchemaType,
+  resetValidators
+};
+