@aifabrix/builder 2.31.1 → 2.32.2

package/lib/schema/external-datasource.schema.json

@@ -2,20 +2,20 @@
    "$schema":"https://json-schema.org/draft/2020-12/schema",
    "$id":"https://raw.githubusercontent.com/esystemsdev/aifabrix-builder/refs/heads/main/lib/schema/external-datasource.schema.json",
    "title":"External Data Source",
-   "description":"Configuration for AI Fabrix ExternalDataSource entities. Includes metadata schema, ABAC access fields, transformation mappings, OpenAPI/MCP exposure, execution logic, and sync behavior.",
+   "description":"Configuration for AI Fabrix ExternalDataSource entities. Includes metadata schema, data dimensions, transformation mappings, OpenAPI/MCP exposure, execution logic, and sync behavior.",
    "metadata":{
       "key":"external-datasource-schema",
       "name":"External Data Source Configuration Schema",
       "description":"JSON schema for validating ExternalDataSource configuration files",
-      "version":"1.1.0",
+      "version":"2.0.0",
       "type":"schema",
       "category":"integration",
       "author":"AI Fabrix Team",
       "createdAt":"2024-01-01T00:00:00Z",
-      "updatedAt":"2025-01-01T00:00:00Z",
+      "updatedAt":"2026-01-18T00:00:00Z",
       "compatibility":{
          "minVersion":"1.0.0",
-         "maxVersion":"2.0.0",
+         "maxVersion":"3.0.0",
          "deprecated":false
       },
       "tags":[
@@ -57,6 +57,29 @@
                "Added data quality rules, indexing, and AI context hints"
             ],
             "breaking":false
+         },
+         {
+            "version":"1.2.0",
+            "date":"2026-01-07T00:00:00Z",
+            "changes":[
+               "Added config.abac section with crossSystemSql and crossSystemJson properties",
+               "Added support for unified filter model (JSON and SQL formats) in ABAC cross-system configuration",
+               "Updated CIP filter expression to support both SQL and JSON formats"
+            ],
+            "breaking":false
+         },
+         {
+            "version":"2.0.0",
+            "date":"2026-01-18T00:00:00Z",
+            "changes":[
+               "BREAKING: Renamed fieldMappings.accessFields (array) to fieldMappings.dimensions (object)",
+               "BREAKING: Renamed fieldMappings.fields to fieldMappings.attributes",
+               "BREAKING: Renamed entityKey to entityType",
+               "BREAKING: Renamed exposed.fields to exposed.attributes",
+               "Added indexed property to attribute definitions for database index creation",
+               "Updated descriptions to reflect dimension-first data model approach"
+            ],
+            "breaking":true
          }
       ]
    },
@@ -65,7 +88,8 @@
       "key",
       "displayName",
       "systemKey",
-      "entityKey",
+      "entityType",
+      "resourceType",
       "fieldMappings"
    ],
    "properties":{
@@ -91,21 +115,16 @@
          "pattern":"^[a-z0-9-]+$",
          "description":"Must match ExternalSystem.key"
       },
-      "entityKey":{
+      "entityType":{
          "type":"string",
-         "pattern":"^[a-z0-9-]+$"
+         "enum":["document-storage","documentStorage","vector-store","vectorStore","record-storage","recordStorage","message-service","messageService","none"],
+         "description":"Entity type identifier. Defines storage semantics and behavior: 'document-storage' or 'documentStorage' (creates document storage service with vector-storage), 'vector-store' or 'vectorStore' (external vector storage system), 'record-storage' or 'recordStorage' (creates record-based system with metadata sync and access rights), 'message-service' or 'messageService' (message service for notifications - Slack, Teams, Email), 'none' (uses external system data directly or connects other data sources). Required field. Supports both camelCase and kebab-case formats for backward compatibility."
       },
       "resourceType":{
          "type":"string",
-         "enum":[
-            "customer",
-            "contact",
-            "person",
-            "document",
-            "deal"
-         ],
+         "pattern":"^[a-z0-9-]+$",
          "default":"document",
-         "description":"Resource type for the datasource. Maps to database ResourceType enum."
+         "description":"Resource type for the datasource. Represents business entity classification. Common values: customer, contact, person, document, deal, record. Any lowercase alphanumeric string with hyphens is allowed."
       },
       "version":{
          "type":"string",
@@ -119,25 +138,24 @@
       },
       "fieldMappings":{
          "type":"object",
-         "description":"Transformation rules and ABAC accessFields. Replaces older flat mapping schema.",
+         "description":"Transformation rules and data dimensions. Maps canonical dimensions to system attributes.",
          "required":[
-            "accessFields",
-            "fields"
+            "dimensions",
+            "attributes"
          ],
          "properties":{
-            "accessFields":{
-               "type":"array",
-               "description":"Normalized fields used in ABAC engine (must exist in 'fields').",
-               "items":{
+            "dimensions":{
+               "type":"object",
+               "description":"Data dimensions mapping. Key = dimension key (from Dimension Catalog), Value = attribute path (e.g., 'metadata.department').",
+               "additionalProperties":{
                   "type":"string",
-                  "pattern":"^[a-zA-Z0-9_]+$"
+                  "pattern":"^[a-zA-Z0-9_.]+$"
                },
-               "minItems":1,
-               "uniqueItems":true
+               "minProperties":0
             },
-            "fields":{
+            "attributes":{
                "type":"object",
-               "description":"Key = normalized field name. Value = transformation expression + type.",
+               "description":"Key = normalized attribute name. Value = transformation expression + type.",
                "additionalProperties":{
                   "type":"object",
                   "required":[
@@ -161,6 +179,11 @@
                            "object"
                         ]
                      },
+                     "indexed":{
+                        "type":"boolean",
+                        "default":false,
+                        "description":"If true, creates database index for fast search/filtering. Dimensions are automatically indexed."
+                     },
                      "description":{
                         "type":"string",
                         "description":"Technical description of the normalized field."
@@ -210,9 +233,9 @@
          "type":"object",
          "description":"Defines which normalized fields are exposed through MCP/OpenAPI. Ensures ISO27001-safe output and predictable AI model behavior.",
          "properties":{
-            "fields":{
+            "attributes":{
                "type":"array",
-               "description":"List of normalized fields (from fieldMappings.fields keys) that the MCP/OpenAPI layers will return.",
+               "description":"List of normalized attributes (from fieldMappings.attributes keys) that the MCP/OpenAPI layers will return.",
                "items":{
                   "type":"string",
                   "pattern":"^[a-zA-Z0-9_]+$"
@@ -270,7 +293,7 @@
             }
          },
          "required":[
-            "fields"
+            "attributes"
          ],
          "additionalProperties":false
       },
@@ -584,112 +607,42 @@
       },
       "documentStorage":{
          "type":"object",
-         "description":"Document storage configuration (optional, enables vector storage)",
+         "description":"Document storage configuration (optional, enables vector storage). Validated against type/document-storage.json schema.",
          "properties":{
             "enabled":{
                "type":"boolean",
                "default":true
             },
-            "flowiseInstanceId":{
-               "type":"string"
-            },
             "twoPhaseSync":{
                "type":"boolean",
-               "default":true
+               "default":true,
+               "description":"Enable two-phase sync pattern. When true: validates metadata first (quality rules, comparison with DocumentRecords), then fetches binaries via CIP for changed/new documents. When false: fetches binaries directly without metadata validation phase (single-phase sync). Note: Files are never synced back to external systems (one-way sync only: external → dataplane)."
+            },
+            "binaryOperationRef":{
+               "type":"string",
+               "default":"get",
+               "description":"CIP operation name for binary document retrieval. Must exist in execution.cip.operations. Defaults to 'get' operation."
             },
-            "binaryOperation":{
+            "responseType":{
+               "type":"string",
+               "enum":[
+                  "binary",
+                  "base64",
+                  "json"
+               ],
+               "default":"binary",
+               "description":"Expected response type from CIP operation. 'binary' for raw binary data, 'base64' for base64-encoded data, 'json' for JSON response with binary field."
+            },
+            "binaryField":{
+               "type":"string",
+               "description":"Field name containing binary data if responseType is 'json' or 'base64'. Required when responseType is not 'binary'."
+            },
+            "parameterMapping":{
                "type":"object",
-               "description":"Configuration for binary document retrieval. Supports CIP operations or direct HTTP/OpenAPI.",
-               "oneOf":[
-                  {
-                     "required":[
-                        "cipOperationRef",
-                        "responseType"
-                     ],
-                     "properties":{
-                        "cipOperationRef":{
-                           "type":"string",
-                           "description":"Reference to CIP operation (e.g., 'get' or custom operation name)"
-                        },
-                        "responseType":{
-                           "type":"string",
-                           "enum":[
-                              "binary",
-                              "base64",
-                              "json"
-                           ],
-                           "default":"binary"
-                        },
-                        "binaryField":{
-                           "type":"string"
-                        },
-                        "parameterMapping":{
-                           "type":"object",
-                           "additionalProperties":{
-                              "type":"string"
-                           }
-                        }
-                     },
-                     "additionalProperties":false
-                  },
-                  {
-                     "required":[
-                        "path",
-                        "responseType"
-                     ],
-                     "properties":{
-                        "operationId":{
-                           "type":"string"
-                        },
-                        "method":{
-                           "type":"string",
-                           "enum":[
-                              "GET",
-                              "POST",
-                              "PUT"
-                           ],
-                           "default":"GET"
-                        },
-                        "path":{
-                           "type":"string"
-                        },
-                        "queryParameters":{
-                           "type":"object",
-                           "additionalProperties":{
-                              "type":"string"
-                           }
-                        },
-                        "headers":{
-                           "type":"object",
-                           "additionalProperties":{
-                              "type":"string"
-                           }
-                        },
-                        "requestBody":{
-                           "type":"object"
-                        },
-                        "responseType":{
-                           "type":"string",
-                           "enum":[
-                              "binary",
-                              "base64",
-                              "json"
-                           ],
-                           "default":"binary"
-                        },
-                        "binaryField":{
-                           "type":"string"
-                        },
-                        "parameterMapping":{
-                           "type":"object",
-                           "additionalProperties":{
-                              "type":"string"
-                           }
-                        }
-                     },
-                     "additionalProperties":false
-                  }
-               ]
+               "additionalProperties":{
+                  "type":"string"
+               },
+               "description":"Map metadata record fields to CIP operation parameters. Example: {\"fileId\": \"{{key}}\", \"downloadUrl\": \"{{metadata.downloadUrl}}\"}"
             },
             "processing":{
                "type":"object",
@@ -715,8 +668,7 @@
             }
          },
          "required":[
-            "flowiseInstanceId",
-            "binaryOperation"
+            "enabled"
          ],
          "additionalProperties":false
       },
@@ -847,6 +799,53 @@
             }
          },
          "additionalProperties":false
+      },
+      "config":{
+         "type":"object",
+         "description":"Additional configuration for this datasource, including ABAC settings, MCP contracts, and other metadata.",
+         "properties":{
+            "abac":{
+               "type":"object",
+               "description":"Attribute-Based Access Control (ABAC) configuration for this datasource.",
+               "properties":{
+                  "dimensions":{
+                     "type":"object",
+                     "description":"Data dimensions mapping. Key = dimension key (from Dimension Catalog), Value = attribute path. Overrides fieldMappings.dimensions if specified.",
+                     "additionalProperties":{
+                        "type":"string",
+                        "pattern":"^[a-zA-Z0-9_.]+$"
+                     }
+                  },
+                  "crossSystemSql":{
+                     "type":"string",
+                     "description":"Cross-system ABAC filter expression in SQL format (advanced, for developers). Example: 'hubspot-companies.country = user.country AND hubspot-companies.revenue >= 1000000'"
+                  },
+                  "crossSystemJson":{
+                     "type":"object",
+                     "description":"Cross-system ABAC filter expression in JSON format (simple, for UI). Example: {'hubspot-companies.country': {'eq': 'user.country'}, 'hubspot-companies.revenue': {'gte': 1000000}}",
+                     "additionalProperties":{
+                        "type":"object",
+                        "properties":{
+                           "eq":{"type":["string","number","boolean"]},
+                           "ne":{"type":["string","number","boolean"]},
+                           "gt":{"type":["string","number"]},
+                           "lt":{"type":["string","number"]},
+                           "gte":{"type":["string","number"]},
+                           "lte":{"type":["string","number"]},
+                           "in":{"type":"array"},
+                           "nin":{"type":"array"},
+                           "contains":{"type":"string"},
+                           "like":{"type":"string"},
+                           "isNull":{"type":"null"},
+                           "isNotNull":{"type":"null"}
+                        }
+                     }
+                  }
+               },
+               "additionalProperties":false
+            }
+         },
+         "additionalProperties":true
       }
    },
    "$defs":{
@@ -1025,7 +1024,10 @@
                      "type":"string",
                      "enum":[
                         "GET",
-                        "POST"
+                        "POST",
+                        "PUT",
+                        "PATCH",
+                        "DELETE"
                      ],
                      "description":"Explicit HTTP method when source='http'."
                   },
@@ -1046,8 +1048,15 @@
                      }
                   },
                   "bodyTemplate":{
-                     "description":"Optional static request body or template for POST/PUT fetches.",
+                     "description":"Optional static request body or template for POST/PUT/PATCH fetches.",
                      "type":["object","string","null"]
+                  },
+                  "headers":{
+                     "type":"object",
+                     "description":"Optional HTTP headers for the request.",
+                     "additionalProperties":{
+                        "type":"string"
+                     }
                   }
                },
                "allOf":[
@@ -1173,7 +1182,7 @@
                   "useFieldMappings":{
                      "type":"boolean",
                      "default":true,
-                     "description":"If true, apply the datasource.fieldMappings.fields expressions; no inline mapping needed."
+                     "description":"If true, apply the datasource.fieldMappings.attributes expressions; no inline mapping needed."
                   },
                   "inline":{
                      "type":"object",
@@ -1200,15 +1209,27 @@
          "properties":{
             "filter":{
                "type":"object",
-               "description":"Filter mapped records before output. ABAC is applied automatically based on fieldMappings.accessFields.",
+               "description":"Filter mapped records before output. Dimension enforcement is applied automatically based on fieldMappings.dimensions.",
                "properties":{
                   "enforceAbac":{
                      "type":"boolean",
                      "default":true
                   },
                   "expression":{
-                     "type":"string",
-                     "description":"Optional additional filter expression (e.g. '@.status == \"active\"')."
+                     "oneOf":[
+                        {
+                           "type":"string",
+                           "description":"SQL filter expression (e.g., 'status = \"active\" AND revenue >= 1000000')"
+                        },
+                        {
+                           "type":"object",
+                           "description":"JSON filter expression (e.g., {'status': {'eq': 'active'}, 'revenue': {'gte': 1000000}})",
+                           "additionalProperties":{
+                              "type":"object"
+                           }
+                        }
+                     ],
+                     "description":"Optional additional filter expression. Supports both SQL and JSON formats."
                   }
                },
                "additionalProperties":false
@@ -1283,4 +1304,5 @@
       }
    },
    "additionalProperties":false
-}
+ }
+ 

package/lib/schema/external-system.schema.json

@@ -12,7 +12,7 @@
     "category": "integration",
     "author": "AI Fabrix Team",
     "createdAt": "2024-01-01T00:00:00Z",
-    "updatedAt": "2024-01-01T00:00:00Z",
+    "updatedAt": "2025-12-01T00:00:00Z",
     "compatibility": {
       "minVersion": "1.0.0",
       "maxVersion": "2.0.0",
@@ -365,6 +365,48 @@
         },
         "additionalProperties": false
       }
+    },
+    "endpoints": {
+      "type": "array",
+      "description": "API endpoint configurations for this external system. Used for dynamic endpoint registration at application startup.",
+      "items": {
+        "type": "object",
+        "required": [
+          "type",
+          "path"
+        ],
+        "properties": {
+          "type": {
+            "type": "string",
+            "description": "Endpoint type identifier (e.g., 'commands', 'events', 'notifications')",
+            "pattern": "^[a-z0-9-]+$"
+          },
+          "path": {
+            "type": "string",
+            "description": "URL path for this endpoint (e.g., '/commands', '/events')",
+            "pattern": "^/[a-z0-9/-]*$"
+          },
+          "description": {
+            "type": "string",
+            "description": "Human-readable description of the endpoint"
+          },
+          "active": {
+            "type": "boolean",
+            "description": "Whether this endpoint should be registered (defaults to true)",
+            "default": true
+          },
+          "router": {
+            "type": "string",
+            "description": "Custom router module path override (defaults to auto-discovery based on naming convention)"
+          }
+        },
+        "additionalProperties": false
+      }
+    },
+    "endpointsActive": {
+      "type": "boolean",
+      "description": "Master switch for all endpoints in this system. If false, no endpoints are registered regardless of individual endpoint active flags.",
+      "default": true
     }
   },
   "additionalProperties": false

package/lib/utils/api.js

@@ -11,7 +11,7 @@
  */
 
 const { parseErrorResponse } = require('./api-error-handler');
-const auditLogger = require('../audit-logger');
+const auditLogger = require('../core/audit-logger');
 
 /**
  * Logs API request performance metrics and errors to audit log
@@ -27,14 +27,18 @@ async function logApiPerformance(params) {
   // Log all API calls (both success and failure) to audit log for troubleshooting
   // This helps track what API calls were made when errors occur
   try {
-    await auditLogger.logApiCall({
+    const logData = {
       url: params.url,
       options: params.options,
       statusCode: params.statusCode,
       duration: params.duration,
-      success: params.success,
-      errorInfo: params.errorInfo || {}
-    });
+      success: params.success
+    };
+    // Only include errorInfo if there's actual error information
+    if (params.errorInfo) {
+      logData.errorInfo = params.errorInfo;
+    }
+    await auditLogger.logApiCall(logData);
   } catch (logError) {
     // Don't fail the API call if audit logging fails
     // Silently continue - audit logging should never break functionality

package/lib/utils/app-register-api.js

@@ -22,6 +22,63 @@ const { formatApiError } = require('./api-error-handler');
  * @param {Object} registrationData - Registration data
  * @returns {Promise<Object>} API response
  */
+/**
+ * Handles registration API error response
+ * @function handleRegistrationError
+ * @param {Object} response - API response
+ * @param {string} apiUrl - Controller URL
+ * @param {Object} registrationData - Registration data
+ */
+function handleRegistrationError(response, apiUrl, registrationData) {
+  const formattedError = response.formattedError || formatApiError(response, apiUrl);
+  logger.error(formattedError);
+  logger.error(chalk.gray(`\nController URL: ${apiUrl}`));
+
+  // For validation errors (400, 422), show the request payload for debugging
+  if (response.status === 400 || response.status === 422) {
+    logger.error(chalk.gray('\nRequest payload:'));
+    logger.error(chalk.gray(JSON.stringify(registrationData, null, 2)));
+    logger.error('');
+    logger.error(chalk.gray('Check your variables.yaml file and ensure all required fields are correctly set.'));
+  }
+
+  process.exit(1);
+}
+
+/**
+ * Extracts application data from API response
+ * @function extractApplicationData
+ * @param {Object} response - API response
+ * @param {string} apiUrl - Controller URL
+ * @returns {Object} Application data
+ * @throws {Error} If response format is invalid
+ */
+function extractApplicationData(response, apiUrl) {
+  const apiResponse = response.data;
+
+  // Handle API response structure:
+  // registerApplication returns: { success: true, data: <API response> }
+  // API response can be:
+  // 1. Direct format: { application: {...}, credentials: {...} }
+  // 2. Wrapped format: { success: true, data: { application: {...}, credentials: {...} } }
+  if (apiResponse && apiResponse.data && apiResponse.data.application) {
+    // Wrapped format: use apiResponse.data
+    return apiResponse.data;
+  }
+
+  if (apiResponse && apiResponse.application) {
+    // Direct format: use apiResponse directly
+    return apiResponse;
+  }
+
+  // Fallback: return apiResponse as-is (shouldn't happen, but handle gracefully)
+  logger.error(chalk.red('❌ Invalid response: missing application data'));
+  logger.error(chalk.gray(`\nController URL: ${apiUrl}`));
+  logger.error(chalk.gray('\nFull response for debugging:'));
+  logger.error(chalk.gray(JSON.stringify(response, null, 2)));
+  process.exit(1);
+}
+
 async function callRegisterApi(apiUrl, token, environment, registrationData) {
   // Use centralized API client
   const authConfig = { type: 'bearer', token: token };
@@ -29,40 +86,11 @@ async function callRegisterApi(apiUrl, token, environment, registrationData) {
     const response = await registerApplication(apiUrl, environment, authConfig, registrationData);
 
     if (!response.success) {
-      const formattedError = response.formattedError || formatApiError(response, apiUrl);
-      logger.error(formattedError);
-      logger.error(chalk.gray(`\nController URL: ${apiUrl}`));
-
-      // For validation errors (400, 422), show the request payload for debugging
-      if (response.status === 400 || response.status === 422) {
-        logger.error(chalk.gray('\nRequest payload:'));
-        logger.error(chalk.gray(JSON.stringify(registrationData, null, 2)));
-        logger.error('');
-        logger.error(chalk.gray('Check your variables.yaml file and ensure all required fields are correctly set.'));
-      }
-
-      process.exit(1);
+      handleRegistrationError(response, apiUrl, registrationData);
+      return; // Never reached, but satisfies linter
     }
 
-    // Handle API response structure:
-    // registerApplication returns: { success: true, data: <API response> }
-    // API response can be:
-    // 1. Direct format: { application: {...}, credentials: {...} }
-    // 2. Wrapped format: { success: true, data: { application: {...}, credentials: {...} } }
-    const apiResponse = response.data;
-    if (apiResponse && apiResponse.data && apiResponse.data.application) {
-      // Wrapped format: use apiResponse.data
-      return apiResponse.data;
-    } else if (apiResponse && apiResponse.application) {
-      // Direct format: use apiResponse directly
-      return apiResponse;
-    }
-    // Fallback: return apiResponse as-is (shouldn't happen, but handle gracefully)
-    logger.error(chalk.red('❌ Invalid response: missing application data'));
-    logger.error(chalk.gray(`\nController URL: ${apiUrl}`));
-    logger.error(chalk.gray('\nFull response for debugging:'));
-    logger.error(chalk.gray(JSON.stringify(response, null, 2)));
-    process.exit(1);
+    return extractApplicationData(response, apiUrl);
   } catch (error) {
     // Include controller URL in error context
     logger.error(chalk.red('❌ Registration API call failed'));