@aifabrix/builder 2.43.0 → 2.44.0

package/integration/hubspot-test/hubspot-test-datasource-users.json

@@ -5,113 +5,72 @@
   "systemKey": "hubspot-test",
   "enabled": true,
   "entityType": "recordStorage",
-  "primaryKey": ["id"],
-  "description": "Comprehensive HubSpot users datasource with full CRUD operations, comprehensive metadata schema, ABAC dimensions, and OpenAPI support",
+  "description": "HubSpot users datasource (minimal v2.4 fixture)",
+  "primaryKey": [
+    "id"
+  ],
+  "labelKey": [
+    "email"
+  ],
   "metadataSchema": {
     "type": "object",
+    "required": [
+      "id",
+      "externalId",
+      "properties"
+    ],
     "properties": {
       "id": {
         "type": "string",
-        "description": "HubSpot user ID"
-      },
-      "properties": {
-        "type": "object",
-        "description": "User properties object containing all CRM fields",
-        "additionalProperties": true,
-        "properties": {
-          "email": {
-            "type": "string",
-            "format": "email",
-            "description": "User email address"
-          },
-          "firstName": {
-            "type": "string",
-            "description": "User first name"
-          },
-          "lastName": {
-            "type": "string",
-            "description": "User last name"
-          },
-          "role": {
-            "type": "string",
-            "description": "User role"
-          },
-          "timezone": {
-            "type": "string",
-            "description": "User timezone"
-          }
-        }
-      },
-      "createdAt": {
-        "type": "string",
-        "format": "date-time",
-        "description": "User creation timestamp"
-      },
-      "updatedAt": {
-        "type": "string",
-        "format": "date-time",
-        "description": "User last update timestamp"
-      },
-      "archived": {
-        "type": "boolean",
-        "description": "Whether the user is archived"
-      },
-      "archivedAt": {
-        "type": "string",
-        "format": "date-time",
-        "description": "User archive timestamp"
-      },
-      "associations": {
-        "type": "object",
-        "description": "Associated records",
-        "additionalProperties": true
+        "index": true
       },
-      "url": {
+      "externalId": {
         "type": "string",
-        "format": "uri",
-        "description": "HubSpot object URL"
+        "index": true
       },
-      "objectWriteTraceId": {
+      "email": {
         "type": "string",
-        "description": "Trace ID for tracking writes"
+        "filter": true
       },
-      "propertiesWithHistory": {
+      "properties": {
         "type": "object",
-        "description": "Properties with change history",
         "additionalProperties": true
       }
-    },
-    "required": [
-      "id"
-    ]
+    }
+  },
+  "dimensions": {
+    "department": {
+      "type": "local",
+      "field": "department"
+    }
   },
   "fieldMappings": {
     "attributes": {
       "id": {
-        "expression": "{{id}}",
-        "type": "string",
-        "indexed": true
+        "expression": "{{raw.id}}"
+      },
+      "externalId": {
+        "expression": "{{raw.id}}"
+      },
+      "email": {
+        "expression": "{{raw.properties.email.value}} | trim"
       },
       "department": {
-        "expression": "{{id}}",
-        "type": "string",
-        "indexed": true,
-        "description": "Default dimension for catalog compatibility"
+        "expression": "{{raw.properties.department.value}} | trim"
       }
-    },
-    "dimensions": {
+    }
+  },
+  "exposed": {
+    "schema": {
+      "id": "metadata.id",
+      "externalId": "metadata.externalId",
+      "email": "metadata.email",
       "department": "metadata.department"
     }
   },
   "sync": {
     "mode": "pull",
     "schedule": "0 */6 * * *",
-    "batchSize": 100,
-    "maxParallelRequests": 5,
-    "pagination": {
-      "strategy": "cursor",
-      "cursorField": "$.paging.next.after",
-      "cursorParam": "after"
-    }
+    "batchSize": 100
   }
-}
+}

package/integration/hubspot-test/hubspot-test-deploy.json

@@ -262,7 +262,7 @@
       "displayName": "HubSpot Company",
       "description": "HubSpot companies datasource with field mappings for CRM company data",
       "systemKey": "hubspot-test",
-      "entityType": "record-storage",
+      "entityType": "recordStorage",
       "resourceType": "customer",
       "enabled": true,
       "version": "1.0.0",
@@ -806,7 +806,7 @@
       "displayName": "HubSpot Contact",
       "description": "HubSpot contacts datasource with field mappings for CRM contact data",
       "systemKey": "hubspot-test",
-      "entityType": "record-storage",
+      "entityType": "recordStorage",
       "resourceType": "contact",
       "enabled": true,
       "version": "1.0.0",
@@ -1448,7 +1448,7 @@
       "displayName": "HubSpot Deal",
       "description": "HubSpot deals datasource with field mappings for CRM deal data",
       "systemKey": "hubspot-test",
-      "entityType": "record-storage",
+      "entityType": "recordStorage",
       "resourceType": "deal",
       "enabled": true,
       "version": "1.0.0",

package/integration/hubspot-test/test-dataplane-down-tests.js

@@ -200,13 +200,7 @@ async function createTestDatasource(datasourcePath) {
  * @returns {string[]} Command arguments
  */
 function buildDatasourceUploadArgs(datasourcePath) {
-  return [
-    'bin/aifabrix.js',
-    'datasource',
-    'upload',
-    'test-app',
-    datasourcePath
-  ];
+  return ['bin/aifabrix.js', 'datasource', 'upload', datasourcePath];
 }
 
 /**

package/integration/hubspot-test/test-dataplane-down.js

@@ -152,15 +152,15 @@ async function runTests() {
       results.push(result);
 
       if (result.success) {
-        logSuccess(`✓ ${result.name}: Error handling validated`);
+        logSuccess(`✔ ${result.name}: Error handling validated`);
       } else {
-        logError(`✗ ${result.name}: Error handling failed`);
+        logError(`✖ ${result.name}: Error handling failed`);
         if (result.output) {
           logWarn(`  Output: ${result.output.substring(0, 200)}...`);
         }
       }
     } catch (error) {
-      logError(`✗ ${testFn.name}: Unexpected error: ${error.message}`);
+      logError(`✖ ${testFn.name}: Unexpected error: ${error.message}`);
       results.push({
         name: testFn.name,
         success: false,

package/integration/hubspot-test/test.js

@@ -430,7 +430,7 @@ async function validateAuth(context, options) {
     throw new Error('Authentication check failed. Run: node bin/aifabrix.js login --controller <url> --method device');
   }
   const output = `${result.stdout}\n${result.stderr}`;
-  // Only treat "Not authenticated" as failure; "✗ Not reachable" (dataplane) is not auth failure
+  // Only treat "Not authenticated" as failure; "✖ Not reachable" (dataplane) is not auth failure
   if (output.includes('Not authenticated')) {
     throw new Error('Not authenticated. Run: node bin/aifabrix.js login --controller <url> --method device');
   }
@@ -1216,7 +1216,7 @@ function buildNegativeCredentialTestCases(context) {
  * @param {string} appName - Application name
  * @param {Object} context - Test context
  * @param {Object} options - Options object
- * @param {string} expectedMessage - Expected error message
+ * @param {string|string[]} expectedMessage - Expected error substring(s); any match passes
  * @returns {Promise<void>} Resolves when validation fails as expected
  * @throws {Error} If validation succeeds or expected message not found
  */
@@ -1228,8 +1228,12 @@ async function runValidationExpectFailure(appName, context, options, expectedMes
   }
   if (expectedMessage) {
     const combined = `${result.stdout}\n${result.stderr}`;
-    if (!combined.includes(expectedMessage)) {
-      throw new Error(`Expected error message not found: ${expectedMessage}\nActual output: ${combined}`);
+    const needles = Array.isArray(expectedMessage) ? expectedMessage : [expectedMessage];
+    const found = needles.some(n => combined.includes(n));
+    if (!found) {
+      throw new Error(
+        `Expected error message not found (any of): ${needles.join(' | ')}\nActual output: ${combined}`
+      );
     }
   }
 }
@@ -1315,20 +1319,16 @@ async function getFirstDatasourcePath(appPath) {
 }
 
 /**
- * Corrupts datasource file by removing dimensions
+ * Corrupts root v2.4 ``dimensions``: local binding without required ``field`` (schema failure).
  * @async
- * @function corruptDatasourceRemoveDimensions
+ * @function corruptDatasourceLocalBindingMissingField
  * @param {string} datasourcePath - Datasource file path
  * @returns {Promise<void>} Resolves when file is corrupted
  */
-async function corruptDatasourceRemoveDimensions(datasourcePath) {
+async function corruptDatasourceLocalBindingMissingField(datasourcePath) {
   const datasourceContent = await fs.readFile(datasourcePath, 'utf8');
   const datasourceJson = JSON.parse(datasourceContent);
-  if (datasourceJson.fieldMappings) {
-    delete datasourceJson.fieldMappings.dimensions;
-  } else {
-    datasourceJson.fieldMappings = {};
-  }
+  datasourceJson.dimensions = { orphan_local: { type: 'local' } };
   await fs.writeFile(datasourcePath, JSON.stringify(datasourceJson, null, 2), 'utf8');
 }
 
@@ -1342,13 +1342,9 @@ async function corruptDatasourceRemoveDimensions(datasourcePath) {
 async function corruptDatasourceInvalidDimensionKey(datasourcePath) {
   const datasourceContent = await fs.readFile(datasourcePath, 'utf8');
   const datasourceJson = JSON.parse(datasourceContent);
-  if (!datasourceJson.fieldMappings) {
-    datasourceJson.fieldMappings = {};
-  }
-  if (!datasourceJson.fieldMappings.dimensions) {
-    datasourceJson.fieldMappings.dimensions = {};
-  }
-  datasourceJson.fieldMappings.dimensions['invalid-key-with-hyphen'] = 'metadata.id';
+  datasourceJson.dimensions = {
+    'invalid-key-with-hyphen': { type: 'local', field: 'id' }
+  };
   await fs.writeFile(datasourcePath, JSON.stringify(datasourceJson, null, 2), 'utf8');
 }
 
@@ -1362,30 +1358,23 @@ async function corruptDatasourceInvalidDimensionKey(datasourcePath) {
 async function corruptDatasourceInvalidAttributePath(datasourcePath) {
   const datasourceContent = await fs.readFile(datasourcePath, 'utf8');
   const datasourceJson = JSON.parse(datasourceContent);
-  if (!datasourceJson.fieldMappings) {
-    datasourceJson.fieldMappings = {};
-  }
-  if (!datasourceJson.fieldMappings.dimensions) {
-    datasourceJson.fieldMappings.dimensions = {};
-  }
-  datasourceJson.fieldMappings.dimensions['valid_key'] = 'metadata.id-with-invalid-chars@';
+  datasourceJson.dimensions = {
+    valid_key: { type: 'local', field: 'id-with-invalid-chars@' }
+  };
   await fs.writeFile(datasourcePath, JSON.stringify(datasourceJson, null, 2), 'utf8');
 }
 
 /**
- * Corrupts datasource file with dimensions as array
+ * Corrupts root ``dimensions`` with a non-object JSON value (schema type error).
  * @async
- * @function corruptDatasourceDimensionsAsArray
+ * @function corruptDatasourceDimensionsWrongJsonType
  * @param {string} datasourcePath - Datasource file path
  * @returns {Promise<void>} Resolves when file is corrupted
  */
-async function corruptDatasourceDimensionsAsArray(datasourcePath) {
+async function corruptDatasourceDimensionsWrongJsonType(datasourcePath) {
   const datasourceContent = await fs.readFile(datasourcePath, 'utf8');
   const datasourceJson = JSON.parse(datasourceContent);
-  if (!datasourceJson.fieldMappings) {
-    datasourceJson.fieldMappings = {};
-  }
-  datasourceJson.fieldMappings.dimensions = ['invalid', 'array', 'format'];
+  datasourceJson.dimensions = ['invalid', 'array', 'format'];
   await fs.writeFile(datasourcePath, JSON.stringify(datasourceJson, null, 2), 'utf8');
 }
 
@@ -1421,7 +1410,10 @@ function buildNegativeRbacTestCases(context) {
         }
         const appPath = await createSystemForNegativeTest(appName, 'wizard-valid-for-rbac-yaml-test', context, options);
         await corruptRbacFile(appPath);
-        await runValidationExpectFailure(appName, context, options, 'Invalid YAML syntax in rbac.yaml');
+        await runValidationExpectFailure(appName, context, options, [
+          'Invalid syntax in rbac.yaml',
+          'Invalid YAML syntax'
+        ]);
         await cleanupAppArtifacts(appName, options);
       }
     }
@@ -1451,23 +1443,23 @@ function buildNegativeDimensionTestCases(context) {
   return [
     createTestCase(
       '2.14',
-      'Datasource missing dimensions in fieldMappings',
+      'Datasource dimensions: local binding missing required field',
       'wizard-e2e-negative-dimension-missing',
       'wizard-valid-for-dimension-test',
-      corruptDatasourceRemoveDimensions,
-      'Missing required property "dimensions"'
+      corruptDatasourceLocalBindingMissingField,
+      'Missing required property "field"'
     ),
     createTestCase(
       '2.15',
-      'Datasource invalid dimension key pattern',
+      'Datasource invalid dimension key (root dimensions propertyNames)',
       'wizard-e2e-negative-dimension-invalid-key',
       'wizard-valid-for-dimension-key-test',
       corruptDatasourceInvalidDimensionKey,
-      'Must be at most 40 characters'
+      ['letters, numbers, and underscores only', 'property name must be valid']
     ),
     createTestCase(
       '2.16',
-      'Datasource invalid attribute path pattern',
+      'Datasource invalid dimension binding field pattern',
       'wizard-e2e-negative-dimension-invalid-path',
       'wizard-valid-for-dimension-path-test',
       corruptDatasourceInvalidAttributePath,
@@ -1475,11 +1467,11 @@ function buildNegativeDimensionTestCases(context) {
     ),
     createTestCase(
       '2.17',
-      'Datasource dimensions as array instead of object',
+      'Datasource dimensions wrong JSON type (array)',
       'wizard-e2e-negative-dimension-array',
       'wizard-valid-for-dimension-array-test',
-      corruptDatasourceDimensionsAsArray,
-      'Expected object, got undefined'
+      corruptDatasourceDimensionsWrongJsonType,
+      'Expected object'
     )
   ];
 }

package/integration/hubspot-test/wizard-hubspot-test-headless.yaml

@@ -0,0 +1,23 @@
+# Headless wizard: regenerate HubSpot integration under integration/hubspot-test/
+# Requires dataplane access (controller → dataplane) and auth (login or client credentials).
+#
+# From repo root:
+#   node bin/aifabrix.js wizard --config integration/hubspot-test/wizard-hubspot-test-headless.yaml \
+#     --controller <CONTROLLER_URL> --environment <ENV>
+#
+# Or set deployment.controller / deployment.environment below and omit CLI flags if your
+# wizard runner merges them (see docs/commands/external-integration.md).
+
+appName: hubspot-test
+mode: create-system
+source:
+  type: known-platform
+  platform: hubspot
+credential:
+  action: skip
+preferences:
+  intent: "HubSpot CRM integration (hubspot-test fixtures)"
+  fieldOnboardingLevel: standard
+deployment:
+  controller: http://localhost:3110
+  environment: dev

package/integration/roundtrip-test-local/README.md

@@ -0,0 +1,144 @@
+# Roundtrip Test
+
+Roundtrip validation test
+
+## System Information
+
+- **System Key**: `roundtrip-test-local`
+- **System Type**: `openapi`
+- **Datasources**: 1
+
+## Files
+
+- `application.yaml` – Application configuration with `app` and `externalIntegration` blocks
+- `roundtrip-test-local-system.yaml` – External system definition (authentication, OpenAPI/MCP, RBAC)
+- `roundtrip-test-local-datasource-roundtrip-test-company.yaml` – Datasource: Company
+- `env.template` – Environment variables template (secrets, API keys)
+- `roundtrip-test-local-deploy.json` – Deployment manifest (generated by `aifabrix json roundtrip-test-local`)
+- `deploy.js` – Deploy script for the integration
+- `wizard.yaml` – Wizard configuration (if created via wizard)
+
+Optional: `rbac.yaml` – Roles and permissions merged into the system when present.
+
+## Quick Start
+
+Login to your controller
+```bash
+aifabrix auth config --set-controller <url> --set-environment dev
+aifabrix login
+```
+
+### 1. Extend External System
+
+Use the interactive wizard to extend your existing system:
+
+```bash
+aifabrix wizard --app roundtrip-test-local
+```
+
+### 2. Configure Authentication and Datasources
+
+Edit files in `integration/roundtrip-test-local/`:
+
+- **Authentication**: `roundtrip-test-local-system.yaml` (auth type, credentials placeholders)
+- **Field mappings**: `roundtrip-test-local-datasource-*-datasource.yaml` (dimensions, attributes, operations)
+- **Credential and configuration**: `env.template` (security settings and configuration variables)
+
+
+### Secrets
+
+Secrets are resolved from `.aifabrix` or key vault. Set them with:
+
+```bash
+aifabrix secret set roundtrip-test-local/apiKey <your value>   # API Key
+```
+
+
+### 3. Validate Configuration
+
+```bash
+aifabrix validate roundtrip-test-local
+```
+
+### 4. Repair Deployment Manifest
+
+**Run repair regularly.** It keeps naming conventions, filenames, and the deployment manifest aligned with AI Fabrix platform best practices. Use it after editing datasources, env.template, or system config—and run it often to catch drift early.
+
+```bash
+aifabrix repair roundtrip-test-local
+```
+
+Options:
+  --auth <method>  Set authentication method (oauth2, aad, apikey, basic, queryParam, oidc, hmac, none); updates system file and env.template
+  --dry-run        Report changes only; do not write
+  --rbac           Ensure RBAC permissions per datasource and add default Admin/Reader roles if none exist
+  --expose         Set exposed.attributes on each datasource to all fieldMappings.attributes keys
+  --sync           Add default sync section to datasources that lack it
+  --test           Generate testPayload.payloadTemplate and testPayload.expectedResult from attributes
+
+### 5. Upload to dataplane
+
+```bash
+aifabrix upload roundtrip-test-local
+```
+
+## Testing
+
+### Unit Tests (Local Validation, No API)
+
+```bash
+aifabrix test roundtrip-test-local
+```
+
+### Integration Tests (Via Dataplane)
+
+```bash
+aifabrix test-integration roundtrip-test-local
+```
+
+### End-to-end Tests (Via Dataplane)
+
+```bash
+aifabrix test-e2e roundtrip-test-local
+```
+
+Options:
+  -e, --env <env>  Environment: dev, tst, or pro (builder: dev/tst for container)
+  -v, --verbose    Show detailed step output and poll progress
+  --debug          Include debug output and write log to integration/roundtrip-test-local/logs/
+  --no-async       Use sync mode (no polling); single POST per datasource
+
+### E2E tests per datasource
+
+To run a full E2E test for a single datasource (config, credential, sync, data, CIP), use `aifabrix datasource test-e2e` with the datasource key and app:
+
+```bash
+# Company
+aifabrix datasource test-e2e roundtrip-test-company --app roundtrip-test-local
+
+```
+
+Options:
+  -a, --app roundtrip-test-local              App key (default: resolve from cwd if inside integration/roundtrip-test-local/)
+  -e, --env <env>                    Environment: dev, tst, or pro
+  -v, --verbose                      Show detailed step output and poll progress
+  --debug                            Include debug output and write log to integration/roundtrip-test-local/logs/
+  --test-crud                        Enable CRUD lifecycle test (body testCrud: true)
+  --record-id <id>                   Record ID for test (body recordId)
+  --no-cleanup                       Disable cleanup after test (body cleanup: false)
+  --primary-key-value <value|@path>  Primary key value or path to JSON file (e.g. @pk.json) for body primaryKeyValue
+  --no-async                         Use sync mode (no polling); single POST, no asyncRun
+
+## Deployment
+
+Deploy via miso-controller pipeline (same as regular apps). Auth and controller come from `aifabrix login` and `aifabrix auth config`:
+
+```bash
+aifabrix deploy roundtrip-test-local
+```
+
+## Troubleshooting
+
+- **Validation errors**: Run `aifabrix validate roundtrip-test-local` to see schema and manifest errors.
+- **Deployment / auth**: Run `aifabrix auth config --set-controller <url> --set-environment <env>` and `aifabrix login` before `aifabrix deploy`.
+- **File not found**: Run commands from the project root (where `package.json` and `integration/` live).

package/integration/roundtrip-test-local/application.yaml

@@ -0,0 +1,13 @@
+app:
+  key: roundtrip-test-local
+  displayName: Roundtrip Test
+  description: Roundtrip validation test
+  type: external
+externalIntegration:
+  schemaBasePath: ./
+  systems:
+    - roundtrip-test-local-system.yaml
+  dataSources:
+    - roundtrip-test-local-datasource-roundtrip-test-company.yaml
+  autopublish: true
+  version: 1.0.0

package/integration/roundtrip-test-local/env.template

@@ -0,0 +1,15 @@
+# Environment variables for external system integration
+# Use kv:// (or aifabrix secret set) for sensitive values; plain values for non-sensitive configuration.
+#
+
+# Authentication
+# Type: oauth2
+KV_ROUNDTRIP_TEST_LOCAL_CLIENTID=kv://roundtrip-test-local/clientId
+KV_ROUNDTRIP_TEST_LOCAL_CLIENTSECRET=kv://roundtrip-test-local/clientSecret
+# Non-secure (e.g. URLs): baseUrl, tokenUrl, authorizationUrl
+
+# Configuration
+# KV_ROUNDTRIP_TEST_CLIENTID
+KV_ROUNDTRIP_TEST_CLIENTID=kv://roundtrip-test/clientId
+# KV_ROUNDTRIP_TEST_CLIENTSECRET
+KV_ROUNDTRIP_TEST_CLIENTSECRET=kv://roundtrip-test/clientSecret

package/integration/roundtrip-test-local/roundtrip-test-local-datasource-roundtrip-test-company.yaml

@@ -0,0 +1,14 @@
+key: roundtrip-test-company
+displayName: Company
+systemKey: roundtrip-test-local
+entityType: record-storage
+resourceType: customer
+primaryKey:
+  - country
+fieldMappings:
+  dimensions:
+    country: metadata.country
+  attributes:
+    country:
+      expression: '{{metadata.country}}'
+      type: string

package/integration/roundtrip-test-local/roundtrip-test-local-deploy.json

@@ -0,0 +1,61 @@
+{
+  "system": {
+    "key": "roundtrip-test-local",
+    "displayName": "Roundtrip Test",
+    "description": "Roundtrip validation test",
+    "type": "openapi",
+    "enabled": true,
+    "authentication": {
+      "method": "oauth2",
+      "variables": {
+        "baseUrl": "https://api.example.com",
+        "tokenUrl": "https://api.example.com/oauth/token",
+        "authorizationUrl": "https://api.example.com/oauth/authorize"
+      },
+      "security": {
+        "clientId": "kv://roundtrip-test/clientId",
+        "clientSecret": "kv://roundtrip-test/clientSecret"
+      }
+    },
+    "configuration": [
+      {
+        "name": "KV_ROUNDTRIP_TEST_CLIENTID",
+        "value": "roundtrip-test/clientId",
+        "location": "keyvault",
+        "required": true
+      },
+      {
+        "name": "KV_ROUNDTRIP_TEST_CLIENTSECRET",
+        "value": "roundtrip-test/clientSecret",
+        "location": "keyvault",
+        "required": true
+      }
+    ],
+    "dataSources": [
+      "roundtrip-test-company"
+    ]
+  },
+  "dataSources": [
+    {
+      "key": "roundtrip-test-company",
+      "displayName": "Company",
+      "systemKey": "roundtrip-test-local",
+      "entityType": "record-storage",
+      "resourceType": "customer",
+      "primaryKey": [
+        "country"
+      ],
+      "fieldMappings": {
+        "dimensions": {
+          "country": "metadata.country"
+        },
+        "attributes": {
+          "country": {
+            "expression": "{{metadata.country}}",
+            "type": "string"
+          }
+        }
+      }
+    }
+  ]
+}