@aifabrix/builder 2.6.3 → 2.8.0

package/lib/schema/external-system.schema.json

@@ -0,0 +1,262 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://raw.githubusercontent.com/esystemsdev/aifabrix-builder/refs/heads/main/lib/schema/external-system.schema.json",
+  "title": "AI Fabrix External System Configuration Schema",
+  "description": "Schema for configuring an external system connected to the AI Fabrix Dataplane. This defines authentication, OpenAPI/MCP bindings, field mappings defaults, metadata handling and portal inputs.",
+
+  "metadata": {
+    "key": "external-system-schema",
+    "name": "External System Configuration Schema",
+    "description": "JSON schema for validating ExternalSystem configuration files",
+    "version": "1.0.0",
+    "type": "schema",
+    "category": "integration",
+    "author": "AI Fabrix Team",
+    "createdAt": "2024-01-01T00:00:00Z",
+    "updatedAt": "2024-01-01T00:00:00Z",
+    "compatibility": {
+      "minVersion": "1.0.0",
+      "maxVersion": "2.0.0",
+      "deprecated": false
+    },
+    "tags": ["schema", "external-system", "dataplane", "integration", "validation"],
+    "dependencies": [],
+    "changelog": [
+      {
+        "version": "1.0.0",
+        "date": "2024-01-01T00:00:00Z",
+        "changes": [
+          "Initial schema for External Systems",
+          "Added OpenAPI and MCP contract configuration",
+          "Added authentication and environment variables",
+          "Added portalInput for UI-driven onboarding",
+          "Compatible with field-mappings.schema and security-model.schema"
+        ],
+        "breaking": false
+      }
+    ]
+  },
+
+  "type": "object",
+
+  "required": [
+    "key",
+    "displayName",
+    "description",
+    "type",
+    "authentication"
+  ],
+
+  "properties": {
+    "key": {
+      "type": "string",
+      "description": "Unique external system identifier (cannot be changed after creation). Example: 'hubspot', 'salesforce', 'teams'.",
+      "pattern": "^[a-z0-9-]+$",
+      "minLength": 3,
+      "maxLength": 40
+    },
+
+    "displayName": {
+      "type": "string",
+      "description": "Human-readable name for the external system",
+      "minLength": 1,
+      "maxLength": 100
+    },
+
+    "description": {
+      "type": "string",
+      "description": "Description of this external system integration",
+      "minLength": 1,
+      "maxLength": 500
+    },
+
+    "type": {
+      "type": "string",
+      "enum": ["openapi", "mcp", "custom"],
+      "description": "Integration type: OpenAPI-driven, MCP-driven, or custom Python connector."
+    },
+
+    "enabled": {
+      "type": "boolean",
+      "default": true
+    },
+
+    "environment": {
+      "type": "object",
+      "description": "Environment-level configuration values used by dataplane and external data sources.",
+      "properties": {
+        "baseUrl": {
+          "type": "string",
+          "description": "Base API URL or MCP server URL",
+          "pattern": "^(http|https)://.*$"
+        },
+        "region": {
+          "type": "string",
+          "description": "Optional region setting for API routing"
+        }
+      },
+      "additionalProperties": true
+    },
+
+    "authentication": {
+      "type": "object",
+      "description": "Authentication configuration for the external system.",
+      "required": ["type"],
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": ["oauth2", "apikey", "basic", "aad", "none"],
+          "description": "Authentication method used for API access."
+        },
+
+        "oauth2": {
+          "type": "object",
+          "description": "OAuth2 configuration used when type == 'oauth2'. All secrets are stored in Key Vault.",
+          "properties": {
+            "tokenUrl": {
+              "type": "string",
+              "pattern": "^(http|https)://.*$"
+            },
+            "clientId": {
+              "type": "string"
+            },
+            "clientSecret": {
+              "type": "string"
+            },
+            "scopes": {
+              "type": "array",
+              "items": { "type": "string" }
+            }
+          },
+          "additionalProperties": false
+        },
+
+        "apikey": {
+          "type": "object",
+          "properties": {
+            "headerName": { "type": "string" },
+            "key": { "type": "string" }
+          },
+          "additionalProperties": false
+        },
+
+        "basic": {
+          "type": "object",
+          "properties": {
+            "username": { "type": "string" },
+            "password": { "type": "string" }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "openapi": {
+      "type": "object",
+      "description": "OpenAPI integration configuration.",
+      "properties": {
+        "documentKey": {
+          "type": "string",
+          "description": "Reference to OpenAPI spec registered via builder. Example: 'hubspot-v3'."
+        },
+        "autoDiscoverEntities": {
+          "type": "boolean",
+          "default": false,
+          "description": "Automatically discover resources/entities from OpenAPI schema."
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "mcp": {
+      "type": "object",
+      "description": "Model Context Protocol integration config.",
+      "properties": {
+        "serverUrl": {
+          "type": "string",
+          "pattern": "^(http|https)://.*$"
+        },
+        "toolPrefix": {
+          "type": "string",
+          "pattern": "^[a-z0-9-]+$",
+          "description": "Prefix for MCP tool names generated from this system."
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "dataSources": {
+      "type": "array",
+      "description": "List of data source keys belonging to this external system. Each must match external-datasource.schema.json.",
+      "items": {
+        "type": "string",
+        "pattern": "^[a-z0-9-]+$"
+      },
+      "uniqueItems": true
+    },
+
+    "configuration": {
+      "type": "array",
+      "description": "External system configuration variables (same pattern as application-schema).",
+      "items": {
+        "type": "object",
+        "required": ["name", "value", "location", "required"],
+        "properties": {
+          "name": {
+            "type": "string",
+            "pattern": "^[A-Z_][A-Z0-9_]*$"
+          },
+          "value": {
+            "type": "string",
+            "description": "Literal value or parameter reference ({{parameter}})"
+          },
+          "location": {
+            "type": "string",
+            "enum": ["variable", "keyvault"]
+          },
+          "required": {
+            "type": "boolean"
+          },
+          "portalInput": {
+            "type": "object",
+            "required": ["field", "label"],
+            "properties": {
+              "field": {
+                "type": "string",
+                "enum": ["password", "text", "textarea", "select", "json"]
+              },
+              "label": { "type": "string" },
+              "placeholder": { "type": "string" },
+              "options": {
+                "type": "array",
+                "items": { "type": "string" }
+              },
+              "masked": { "type": "boolean" },
+              "validation": {
+                "type": "object",
+                "properties": {
+                  "minLength": { "type": "integer" },
+                  "maxLength": { "type": "integer" },
+                  "pattern": { "type": "string" },
+                  "required": { "type": "boolean" }
+                },
+                "additionalProperties": false
+              }
+            },
+            "additionalProperties": false
+          }
+        },
+        "additionalProperties": false
+      }
+    },
+
+    "tags": {
+      "type": "array",
+      "description": "Optional tags for search / filtering",
+      "items": { "type": "string" }
+    }
+  },
+
+  "additionalProperties": false
+}

package/lib/schema/infrastructure-schema.json

@@ -1,6 +1,6 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
-  "$id": "https://docs.aifabrix.ai/schemas/Configuration-Schema-Infra.json",
+  "$id": "https://raw.githubusercontent.com/esystemsdev/aifabrix-builder/refs/heads/main/lib/schema/infrastructure-schema.json",
   "title": "AI Fabrix Infrastructure Configuration Schema",
   "description": "Infrastructure-level configuration schema for AI Fabrix deployment",
   "metadata": {

package/lib/secrets.js

@@ -20,7 +20,6 @@ const {
   formatMissingSecretsFileInfo,
   replaceKvInContent,
   loadEnvTemplate,
-  processEnvVariables,
   adjustLocalEnvPortsInContent,
   rewriteInfraEndpoints,
   readYamlAtPath,
@@ -28,6 +27,7 @@ const {
   ensureNonEmptySecrets,
   validateSecrets
 } = require('./utils/secrets-helpers');
+const { processEnvVariables } = require('./utils/env-copy');
 const { buildEnvVarMap } = require('./utils/env-map');
 const { resolveServicePortsInEnvContent } = require('./utils/secrets-url');
 const {
@@ -291,6 +291,25 @@ async function applyEnvironmentTransformations(resolved, environment, variablesP
   if (environment === 'docker') {
     resolved = await resolveServicePortsInEnvContent(resolved, environment);
     resolved = await rewriteInfraEndpoints(resolved, 'docker');
+    // Interpolate ${VAR} references created by rewriteInfraEndpoints
+    // Get the actual host and port values from env-endpoints.js directly
+    // to ensure they are correctly populated in envVars for interpolation
+    const { getEnvHosts, getServiceHost, getServicePort, getLocalhostOverride } = require('./utils/env-endpoints');
+    const hosts = await getEnvHosts('docker');
+    const localhostOverride = getLocalhostOverride('docker');
+    const redisHost = getServiceHost(hosts.REDIS_HOST, 'docker', 'redis', localhostOverride);
+    const redisPort = await getServicePort('REDIS_PORT', 'redis', hosts, 'docker', null);
+    const dbHost = getServiceHost(hosts.DB_HOST, 'docker', 'postgres', localhostOverride);
+    const dbPort = await getServicePort('DB_PORT', 'postgres', hosts, 'docker', null);
+
+    // Build envVars map and ensure it has the correct values
+    const envVars = await buildEnvVarMap('docker');
+    // Override with the actual values that were just set by rewriteInfraEndpoints
+    envVars.REDIS_HOST = redisHost;
+    envVars.REDIS_PORT = String(redisPort);
+    envVars.DB_HOST = dbHost;
+    envVars.DB_PORT = String(dbPort);
+    resolved = interpolateEnvVars(resolved, envVars);
     resolved = await updatePortForDocker(resolved, variablesPath);
   } else if (environment === 'local') {
     // adjustLocalEnvPortsInContent handles both PORT and infra endpoints

package/lib/templates.js

@@ -16,6 +16,37 @@ const yaml = require('js-yaml');
  */
 function generateVariablesYaml(appName, config) {
   const displayName = appName.replace(/-/g, ' ').replace(/\b\w/g, l => l.toUpperCase());
+  const appType = config.type || 'webapp';
+
+  // For external type, create minimal variables.yaml
+  if (appType === 'external') {
+    const variables = {
+      app: {
+        key: appName,
+        displayName: displayName,
+        description: `${appName.replace(/-/g, ' ')} external system`,
+        type: 'external'
+      },
+      deployment: {
+        controllerUrl: '',
+        environment: 'dev'
+      },
+      externalIntegration: {
+        schemaBasePath: './schemas',
+        systems: [],
+        dataSources: [],
+        autopublish: true,
+        version: '1.0.0'
+      }
+    };
+
+    return yaml.dump(variables, {
+      indent: 2,
+      lineWidth: 120,
+      noRefs: true,
+      sortKeys: false
+    });
+  }
 
   // Parse image name to separate name and tag
   let imageName = appName;
@@ -32,7 +63,7 @@ function generateVariablesYaml(appName, config) {
       key: appName,
       displayName: displayName,
       description: `${appName.replace(/-/g, ' ')} application`,
-      type: 'webapp'
+      type: appType
     },
     image: {
       name: imageName,

package/lib/utils/device-code.js

@@ -61,20 +61,28 @@ function parseDeviceCodeResponse(response) {
  * @function initiateDeviceCodeFlow
  * @param {string} controllerUrl - Base URL of the controller
  * @param {string} environment - Environment key (e.g., 'miso', 'dev', 'tst', 'pro')
+ * @param {string} [scope] - OAuth2 scope string (default: 'openid profile email')
  * @returns {Promise<Object>} Device code response with device_code, user_code, verification_uri, expires_in, interval
  * @throws {Error} If initiation fails
  */
-async function initiateDeviceCodeFlow(controllerUrl, environment) {
+async function initiateDeviceCodeFlow(controllerUrl, environment, scope) {
   if (!environment || typeof environment !== 'string') {
     throw new Error('Environment key is required');
   }
 
+  // Default scope for backward compatibility
+  const defaultScope = 'openid profile email';
+  const requestScope = scope || defaultScope;
+
   const url = `${controllerUrl}/api/v1/auth/login?environment=${encodeURIComponent(environment)}`;
   const response = await getMakeApiCall()(url, {
     method: 'POST',
     headers: {
       'Content-Type': 'application/json'
-    }
+    },
+    body: JSON.stringify({
+      scope: requestScope
+    })
   });
 
   if (!response.success) {

package/lib/utils/env-copy.js

@@ -16,6 +16,8 @@ const logger = require('./logger');
 const config = require('../config');
 const devConfig = require('../utils/dev-config');
 const { rewriteInfraEndpoints } = require('./env-endpoints');
+const { buildEnvVarMap } = require('./env-map');
+const { interpolateEnvVars } = require('./secrets-helpers');
 
 /**
  * Process and optionally copy env file to envOutputPath if configured
@@ -120,6 +122,28 @@ async function processEnvVariables(envPath, variablesPath, appName, secretsPath)
     });
     // Rewrite infra endpoints using env-config mapping for local context
     envContent = await rewriteInfraEndpoints(envContent, 'local', infraPorts);
+    // Interpolate ${VAR} references created by rewriteInfraEndpoints
+    // Extract actual values from updated content to use for interpolation
+    const envVars = await buildEnvVarMap('local', null, devIdNum);
+    // Extract REDIS_HOST, REDIS_PORT, DB_HOST, DB_PORT from updated content if present
+    // Only extract if the value doesn't contain ${VAR} references (to avoid circular interpolation)
+    const redisHostMatch = envContent.match(/^REDIS_HOST\s*=\s*([^\r\n$]+)/m);
+    const redisPortMatch = envContent.match(/^REDIS_PORT\s*=\s*([^\r\n$]+)/m);
+    const dbHostMatch = envContent.match(/^DB_HOST\s*=\s*([^\r\n$]+)/m);
+    const dbPortMatch = envContent.match(/^DB_PORT\s*=\s*([^\r\n$]+)/m);
+    if (redisHostMatch && redisHostMatch[1] && !redisHostMatch[1].includes('${')) {
+      envVars.REDIS_HOST = redisHostMatch[1].trim();
+    }
+    if (redisPortMatch && redisPortMatch[1] && !redisPortMatch[1].includes('${')) {
+      envVars.REDIS_PORT = redisPortMatch[1].trim();
+    }
+    if (dbHostMatch && dbHostMatch[1] && !dbHostMatch[1].includes('${')) {
+      envVars.DB_HOST = dbHostMatch[1].trim();
+    }
+    if (dbPortMatch && dbPortMatch[1] && !dbPortMatch[1].includes('${')) {
+      envVars.DB_PORT = dbPortMatch[1].trim();
+    }
+    envContent = interpolateEnvVars(envContent, envVars);
     fs.writeFileSync(outputPath, envContent, { mode: 0o600 });
     logger.log(chalk.green(`✓ Copied .env to: ${variables.build.envOutputPath}`));
   }

package/lib/utils/env-endpoints.js

@@ -149,27 +149,26 @@ function updateEndpointVariables(envContent, redisHost, redisPort, dbHost, dbPor
 
   // Update REDIS_URL if present
   if (/^REDIS_URL\s*=.*$/m.test(updated)) {
-    const m = updated.match(/^REDIS_URL\s*=\s*redis:\/\/([^:\s]+):\d+/m);
-    const currentHost = m && m[1] ? m[1] : null;
-    const targetHost = redisHost || currentHost;
-    if (targetHost) {
-      updated = updated.replace(
-        /^REDIS_URL\s*=\s*.*$/m,
-        `REDIS_URL=redis://${targetHost}:${redisPort}`
-      );
-    }
+    updated = updated.replace(
+      /^REDIS_URL\s*=\s*.*$/m,
+      'REDIS_URL=redis://${REDIS_HOST}:${REDIS_PORT}'
+    );
   }
 
   // Update REDIS_HOST if present
+  // If the original has host:port pattern, use ${VAR} references
+  // Otherwise, just set the host value
   if (/^REDIS_HOST\s*=.*$/m.test(updated)) {
     const hostPortMatch = updated.match(/^REDIS_HOST\s*=\s*([a-zA-Z0-9_.-]+):\d+$/m);
     const hasPortPattern = !!hostPortMatch;
     if (hasPortPattern) {
+      // Original had host:port pattern, use ${VAR} references
       updated = updated.replace(
         /^REDIS_HOST\s*=\s*.*$/m,
-        `REDIS_HOST=${redisHost}:${redisPort}`
+        'REDIS_HOST=${REDIS_HOST}:${REDIS_PORT}'
       );
     } else {
+      // Just host, set actual value
       updated = updated.replace(/^REDIS_HOST\s*=\s*.*$/m, `REDIS_HOST=${redisHost}`);
     }
   }
@@ -195,6 +194,31 @@ function updateEndpointVariables(envContent, redisHost, redisPort, dbHost, dbPor
     );
   }
 
+  // Update DATABASE_URL and other database URL variables (postgresql:// or postgres:// URLs)
+  // Handles DATABASE_URL, DATABASELOG_URL, and any other *_URL variables with postgresql:// or postgres://
+  // First, collect all matches to avoid issues with modifying string during iteration
+  const dbUrlPattern = /^(DATABASE[^=]*_URL)\s*=\s*(postgresql?:\/\/)([^:]+):([^@]+)@([^:/]+):(\d+)(\/[^\s]*)?/gm;
+  const matches = [];
+  let dbUrlMatch;
+  // Reset regex lastIndex to ensure we start from beginning
+  dbUrlPattern.lastIndex = 0;
+  while ((dbUrlMatch = dbUrlPattern.exec(updated)) !== null) {
+    matches.push({
+      varName: dbUrlMatch[1], // DATABASE_URL, DATABASELOG_URL, etc.
+      protocol: dbUrlMatch[2], // postgresql:// or postgres://
+      user: dbUrlMatch[3], // username
+      password: dbUrlMatch[4], // password
+      path: dbUrlMatch[7] || '' // /database path
+    });
+  }
+  // Now apply all replacements
+  for (const match of matches) {
+    updated = updated.replace(
+      new RegExp(`^${match.varName}\\s*=\\s*.*$`, 'm'),
+      `${match.varName}=${match.protocol}${match.user}:${match.password}@\${DB_HOST}:\${DB_PORT}${match.path}`
+    );
+  }
+
   // Update DATABASE_PORT if present (some templates use DATABASE_PORT instead of DB_PORT)
   if (/^DATABASE_PORT\s*=.*$/m.test(updated)) {
     updated = updated.replace(
@@ -203,6 +227,20 @@ function updateEndpointVariables(envContent, redisHost, redisPort, dbHost, dbPor
     );
   }
 
+  // Update Keycloak database variables if present
+  // KC_DB_URL_HOST is used by Keycloak to connect to the database
+  if (/^KC_DB_URL_HOST\s*=.*$/m.test(updated)) {
+    updated = updated.replace(/^KC_DB_URL_HOST\s*=\s*.*$/m, `KC_DB_URL_HOST=${dbHost}`);
+  }
+
+  // KC_DB_URL_PORT is used by Keycloak for database port
+  if (/^KC_DB_URL_PORT\s*=.*$/m.test(updated)) {
+    updated = updated.replace(
+      /^KC_DB_URL_PORT\s*=\s*.*$/m,
+      `KC_DB_URL_PORT=${dbPort}`
+    );
+  }
+
   return updated;
 }
 
@@ -259,6 +297,7 @@ module.exports = {
   splitHost,
   getServicePort,
   getServiceHost,
-  updateEndpointVariables
+  updateEndpointVariables,
+  getLocalhostOverride
 };