@aifabrix/builder 2.41.0 → 2.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/.cursor/rules/docs-rules.mdc +30 -0
  2. package/README.md +1 -1
  3. package/integration/hubspot/README.md +8 -4
  4. package/integration/hubspot/application.json +54 -0
  5. package/integration/hubspot/create-hubspot.js +9 -136
  6. package/integration/hubspot/env.template +3 -4
  7. package/integration/hubspot/hubspot-datasource-company.json +343 -5
  8. package/integration/hubspot/hubspot-datasource-contact.json +413 -5
  9. package/integration/hubspot/hubspot-datasource-deal.json +341 -4
  10. package/integration/hubspot/hubspot-datasource-users.json +116 -0
  11. package/integration/hubspot/hubspot-deploy.json +1250 -108
  12. package/integration/hubspot/hubspot-system.json +15 -32
  13. package/integration/hubspot/test-dataplane-down-tests.js +17 -16
  14. package/integration/hubspot/test-dataplane-down.js +2 -2
  15. package/jest.config.manual.js +2 -1
  16. package/lib/api/external-test.api.js +111 -0
  17. package/lib/api/index.js +42 -19
  18. package/lib/api/pipeline.api.js +66 -120
  19. package/lib/api/types/pipeline.types.js +37 -0
  20. package/lib/api/wizard-platform.api.js +61 -0
  21. package/lib/api/wizard.api.js +34 -1
  22. package/lib/app/config.js +23 -11
  23. package/lib/app/index.js +3 -1
  24. package/lib/app/prompts.js +44 -29
  25. package/lib/app/readme.js +8 -3
  26. package/lib/app/run-env-compose.js +64 -1
  27. package/lib/app/run-helpers.js +1 -1
  28. package/lib/app/show-display.js +1 -1
  29. package/lib/cli/setup-app.js +42 -11
  30. package/lib/cli/setup-credential-deployment.js +31 -6
  31. package/lib/cli/setup-dev.js +27 -0
  32. package/lib/cli/setup-environment.js +12 -4
  33. package/lib/cli/setup-external-system.js +19 -4
  34. package/lib/cli/setup-infra.js +54 -14
  35. package/lib/cli/setup-utility.js +117 -21
  36. package/lib/commands/credential-env.js +162 -0
  37. package/lib/commands/credential-list.js +17 -22
  38. package/lib/commands/credential-push.js +96 -0
  39. package/lib/commands/datasource.js +77 -6
  40. package/lib/commands/dev-init.js +39 -1
  41. package/lib/commands/repair-auth-config.js +99 -0
  42. package/lib/commands/repair-datasource-keys.js +208 -0
  43. package/lib/commands/repair-datasource.js +235 -0
  44. package/lib/commands/repair-env-template.js +348 -0
  45. package/lib/commands/repair-internal.js +85 -0
  46. package/lib/commands/repair-rbac.js +158 -0
  47. package/lib/commands/repair.js +507 -0
  48. package/lib/commands/test-e2e-external.js +165 -0
  49. package/lib/commands/upload.js +71 -40
  50. package/lib/commands/wizard-core-helpers.js +226 -4
  51. package/lib/commands/wizard-core.js +67 -29
  52. package/lib/commands/wizard-dataplane.js +1 -1
  53. package/lib/commands/wizard-entity-selection.js +43 -0
  54. package/lib/commands/wizard-headless.js +44 -5
  55. package/lib/commands/wizard-helpers.js +7 -3
  56. package/lib/commands/wizard.js +86 -64
  57. package/lib/core/config.js +7 -1
  58. package/lib/core/secrets.js +33 -12
  59. package/lib/datasource/deploy.js +12 -3
  60. package/lib/datasource/test-e2e.js +219 -0
  61. package/lib/datasource/test-integration.js +154 -0
  62. package/lib/deployment/deployer.js +7 -5
  63. package/lib/external-system/download.js +182 -204
  64. package/lib/external-system/generator.js +204 -56
  65. package/lib/external-system/test-execution.js +2 -1
  66. package/lib/external-system/test-system-level.js +73 -0
  67. package/lib/external-system/test.js +51 -18
  68. package/lib/generator/external-controller-manifest.js +29 -2
  69. package/lib/generator/external-schema-utils.js +1 -1
  70. package/lib/generator/external.js +10 -3
  71. package/lib/generator/index.js +4 -1
  72. package/lib/generator/split-readme.js +1 -0
  73. package/lib/generator/split-variables.js +7 -1
  74. package/lib/generator/split.js +194 -54
  75. package/lib/generator/wizard-prompts-secondary.js +294 -0
  76. package/lib/generator/wizard-prompts.js +105 -106
  77. package/lib/generator/wizard-readme.js +88 -0
  78. package/lib/generator/wizard.js +147 -158
  79. package/lib/infrastructure/compose.js +11 -1
  80. package/lib/infrastructure/index.js +11 -3
  81. package/lib/infrastructure/services.js +22 -11
  82. package/lib/schema/application-schema.json +8 -5
  83. package/lib/schema/external-datasource.schema.json +49 -26
  84. package/lib/schema/external-system.schema.json +82 -6
  85. package/lib/schema/wizard-config.schema.json +16 -0
  86. package/lib/utils/api.js +38 -10
  87. package/lib/utils/auth-headers.js +8 -7
  88. package/lib/utils/compose-generator.js +1 -1
  89. package/lib/utils/compose-handlebars-helpers.js +11 -0
  90. package/lib/utils/config-format-preference.js +51 -0
  91. package/lib/utils/config-format.js +36 -0
  92. package/lib/utils/configuration-env-resolver.js +179 -0
  93. package/lib/utils/credential-display.js +83 -0
  94. package/lib/utils/credential-secrets-env.js +115 -25
  95. package/lib/utils/dataplane-pipeline-warning.js +28 -0
  96. package/lib/utils/deployment-validation-helpers.js +4 -4
  97. package/lib/utils/dev-ca-install.js +139 -0
  98. package/lib/utils/env-copy.js +23 -3
  99. package/lib/utils/error-formatters/http-status-errors.js +0 -1
  100. package/lib/utils/error-formatters/permission-errors.js +0 -1
  101. package/lib/utils/error-formatters/validation-errors.js +0 -1
  102. package/lib/utils/external-readme.js +56 -29
  103. package/lib/utils/external-system-display.js +59 -1
  104. package/lib/utils/external-system-test-helpers.js +21 -8
  105. package/lib/utils/external-system-validators.js +3 -0
  106. package/lib/utils/file-upload.js +20 -50
  107. package/lib/utils/help-builder.js +1 -0
  108. package/lib/utils/infra-status.js +50 -44
  109. package/lib/utils/local-secrets.js +5 -5
  110. package/lib/utils/paths.js +85 -4
  111. package/lib/utils/secrets-canonical.js +93 -0
  112. package/lib/utils/secrets-generator.js +20 -0
  113. package/lib/utils/secrets-helpers.js +75 -89
  114. package/lib/utils/test-log-writer.js +56 -0
  115. package/lib/utils/token-manager.js +24 -32
  116. package/lib/validation/env-template-auth.js +157 -0
  117. package/lib/validation/env-template-kv.js +41 -0
  118. package/lib/validation/external-manifest-validator.js +25 -0
  119. package/lib/validation/external-system-auth-rules.js +86 -0
  120. package/lib/validation/validate-batch.js +149 -0
  121. package/lib/validation/validate-datasource-keys-api.js +33 -0
  122. package/lib/validation/validate-display.js +94 -16
  123. package/lib/validation/validate.js +25 -12
  124. package/lib/validation/validator.js +7 -9
  125. package/lib/validation/wizard-datasource-validation.js +50 -0
  126. package/package.json +7 -2
  127. package/templates/applications/dataplane/application.yaml +1 -1
  128. package/templates/applications/dataplane/env.template +5 -5
  129. package/templates/applications/dataplane/rbac.yaml +2 -2
  130. package/templates/applications/miso-controller/env.template +1 -1
  131. package/templates/external-system/README.md.hbs +65 -25
  132. package/templates/external-system/deploy.js.hbs +4 -2
  133. package/templates/external-system/external-datasource.yaml.hbs +217 -0
  134. package/templates/external-system/external-system.json.hbs +1 -18
  135. package/templates/infra/compose.yaml.hbs +6 -0
  136. package/templates/python/docker-compose.hbs +4 -4
  137. package/templates/typescript/docker-compose.hbs +4 -4
  138. package/integration/hubspot/application.yaml +0 -37
@@ -16,11 +16,16 @@ const { resolveExternalFiles } = require('../utils/schema-resolver');
16
16
  const { loadExternalSystemSchema, loadExternalDataSourceSchema, detectSchemaType } = require('../utils/schema-loader');
17
17
  const { formatValidationErrors } = require('../utils/error-formatter');
18
18
  const { detectAppType } = require('../utils/paths');
19
+ const batch = require('./validate-batch');
19
20
  const { logOfflinePathWhenType } = require('../utils/cli-utils');
20
21
  const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
21
- const { displayValidationResults } = require('./validate-display');
22
+ const { displayValidationResults, displayBatchValidationResults } = require('./validate-display');
22
23
  const { generateControllerManifest } = require('../generator/external-controller-manifest');
23
24
  const { validateControllerManifest } = require('./external-manifest-validator');
25
+ const {
26
+ validateOAuth2GrantTypeAndAuthorizationUrl,
27
+ validateConfigurationNoStandardAuthVariables
28
+ } = require('./external-system-auth-rules');
24
29
 
25
30
  /**
26
31
  * Validates a file path (detects type and validates)
@@ -191,6 +196,8 @@ async function validateExternalFile(filePath, type) {
191
196
 
192
197
  if (normalizedType === 'system') {
193
198
  validateRoleReferences(parseResult.parsed, errors);
199
+ validateOAuth2GrantTypeAndAuthorizationUrl(parseResult.parsed, errors);
200
+ validateConfigurationNoStandardAuthVariables(parseResult.parsed, errors);
194
201
  }
195
202
 
196
203
  return {
@@ -398,6 +405,7 @@ async function validateExternalSystemComplete(appName, options = {}) {
398
405
  throw new Error('App name is required and must be a string');
399
406
  }
400
407
 
408
+ const { appPath } = await detectAppType(appName, options);
401
409
  const steps = {
402
410
  application: { valid: false, errors: [], warnings: [] },
403
411
  components: { valid: false, errors: [], warnings: [], files: [] },
@@ -414,11 +422,13 @@ async function validateExternalSystemComplete(appName, options = {}) {
414
422
 
415
423
  // If components have errors, return early (don't validate manifest)
416
424
  if (!steps.components.valid) {
425
+ steps.manifest = { valid: true, errors: [], warnings: [], skipped: true };
417
426
  return {
418
427
  valid: false,
419
428
  errors: [...steps.application.errors, ...steps.components.errors],
420
429
  warnings: [...steps.application.warnings, ...steps.components.warnings],
421
- steps
430
+ steps,
431
+ appPath
422
432
  };
423
433
  }
424
434
 
@@ -433,7 +443,8 @@ async function validateExternalSystemComplete(appName, options = {}) {
433
443
  valid: allErrors.length === 0,
434
444
  errors: allErrors,
435
445
  warnings: allWarnings,
436
- steps
446
+ steps,
447
+ appPath
437
448
  };
438
449
  }
439
450
 
@@ -451,29 +462,31 @@ async function validateAppOrFile(appOrFile, options = {}) {
451
462
  const { appPath, isExternal } = await detectAppType(appName);
452
463
  logOfflinePathWhenType(appPath);
453
464
 
454
- if (isExternal) {
455
- return await validateExternalSystemComplete(appName, options);
456
- }
465
+ if (isExternal) return await validateExternalSystemComplete(appName, options);
457
466
 
458
467
  const appValidation = await validator.validateApplication(appName, options);
459
468
  const rbacValidation = await validateRbacForExternalSystem(isExternal, appName);
460
-
461
469
  const variablesPath = resolveApplicationConfigPath(appPath);
462
470
  const earlyReturn = loadVariablesAndCheckExternalIntegration(variablesPath, appValidation);
463
- if (earlyReturn) {
464
- return earlyReturn;
465
- }
471
+ if (earlyReturn) return earlyReturn;
466
472
 
467
473
  const externalValidations = await validateExternalFilesForApp(appName, options);
468
- return aggregateValidationResults(appValidation, externalValidations, rbacValidation);
474
+ const result = aggregateValidationResults(appValidation, externalValidations, rbacValidation);
475
+ result.appPath = appPath;
476
+ return result;
469
477
  }
470
478
 
471
479
  module.exports = {
472
480
  validateAppOrFile,
473
481
  validateExternalSystemComplete,
474
482
  displayValidationResults,
483
+ displayBatchValidationResults,
475
484
  validateExternalFile,
476
485
  validateExternalFilesForApp,
477
- validateFilePath
486
+ validateFilePath,
487
+ validateAllIntegrations: (opts = {}) => batch.validateAllIntegrations(validateAppOrFile, opts),
488
+ validateAllBuilderApps: (opts = {}) => batch.validateAllBuilderApps(validateAppOrFile, opts),
489
+ validateAll: (opts = {}) => batch.validateAll(validateAppOrFile, opts),
490
+ buildBatchResult: batch.buildBatchResult
478
491
  };
479
492
 
@@ -21,6 +21,8 @@ const { checkEnvironment } = require('../utils/environment-checker');
21
21
  const { formatValidationErrors } = require('../utils/error-formatter');
22
22
  const { detectAppType, resolveApplicationConfigPath } = require('../utils/paths');
23
23
  const { loadConfigFile } = require('../utils/config-format');
24
+ const { validateAuthKvCoverage } = require('./env-template-auth');
25
+ const { validateKvReferencesInLines } = require('./env-template-kv');
24
26
 
25
27
  /**
26
28
  * Validates application config file against application schema
@@ -250,7 +252,7 @@ async function validateEnvTemplate(appName, options = {}) {
250
252
  }
251
253
 
252
254
  // Support both builder/ and integration/ directories using detectAppType
253
- const { appPath } = await detectAppType(appName, options);
255
+ const { appPath, isExternal } = await detectAppType(appName, options);
254
256
  const templatePath = path.join(appPath, 'env.template');
255
257
 
256
258
  if (!fs.existsSync(templatePath)) {
@@ -281,14 +283,10 @@ async function validateEnvTemplate(appName, options = {}) {
281
283
  }
282
284
  });
283
285
 
284
- // Check for kv:// reference format
285
- const kvPattern = /kv:\/\/([a-zA-Z0-9-_]+)/g;
286
- let match;
287
- while ((match = kvPattern.exec(content)) !== null) {
288
- const secretKey = match[1];
289
- if (!secretKey) {
290
- errors.push('Invalid kv:// reference format');
291
- }
286
+ validateKvReferencesInLines(lines, errors);
287
+
288
+ if (isExternal) {
289
+ await validateAuthKvCoverage(appPath, content, errors, warnings, options);
292
290
  }
293
291
 
294
292
  return {
@@ -0,0 +1,50 @@
1
+ /**
2
+ * @fileoverview Wizard datasource validation helpers - validate datasourceKeys and entityName against dataplane
3
+ * @author AI Fabrix Team
4
+ * @version 2.0.0
5
+ */
6
+
7
+ /**
8
+ * Validate that all datasourceKeys exist in the platform's available datasources
9
+ * @function validateDatasourceKeysForPlatform
10
+ * @param {string[]} datasourceKeys - User-provided datasource keys
11
+ * @param {Array<{key: string, displayName?: string, entity?: string}>} availableDatasources - Platform datasources
12
+ * @returns {{ valid: boolean, invalidKeys: string[] }} Validation result
13
+ */
14
+ function validateDatasourceKeysForPlatform(datasourceKeys, availableDatasources) {
15
+ if (!Array.isArray(datasourceKeys) || datasourceKeys.length === 0) {
16
+ return { valid: true, invalidKeys: [] };
17
+ }
18
+ const availableKeys = Array.isArray(availableDatasources)
19
+ ? availableDatasources.map(d => (d && typeof d === 'object' && d.key) ? d.key : null).filter(Boolean)
20
+ : [];
21
+ const invalidKeys = datasourceKeys.filter(k => !availableKeys.includes(k));
22
+ return {
23
+ valid: invalidKeys.length === 0,
24
+ invalidKeys
25
+ };
26
+ }
27
+
28
+ /**
29
+ * Validate that entityName exists in the discovered entities list
30
+ * @function validateEntityNameForOpenApi
31
+ * @param {string} entityName - User-provided entity name
32
+ * @param {Array<{name: string}>} entities - Discovered entities from OpenAPI
33
+ * @returns {{ valid: boolean }} Validation result
34
+ */
35
+ function validateEntityNameForOpenApi(entityName, entities) {
36
+ if (!entityName || typeof entityName !== 'string' || entityName.trim() === '') {
37
+ return { valid: true };
38
+ }
39
+ const entityNames = Array.isArray(entities)
40
+ ? entities.map(e => (e && typeof e === 'object' && e.name) ? e.name : null).filter(Boolean)
41
+ : [];
42
+ return {
43
+ valid: entityNames.includes(entityName)
44
+ };
45
+ }
46
+
47
+ module.exports = {
48
+ validateDatasourceKeysForPlatform,
49
+ validateEntityNameForOpenApi
50
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@aifabrix/builder",
3
- "version": "2.41.0",
3
+ "version": "2.42.0",
4
4
  "description": "AI Fabrix Local Fabric & Deployment SDK",
5
5
  "main": "lib/index.js",
6
6
  "bin": {
@@ -10,6 +10,7 @@
10
10
  "scripts": {
11
11
  "test": "node tests/scripts/test-wrapper.js",
12
12
  "test:ci": "bash tests/scripts/ci-simulate.sh",
13
+ "test:same-as-github": "cross-env CI=true npm run lint && cross-env CI=true node tests/scripts/test-wrapper.js",
13
14
  "test:coverage": "cross-env RUN_COVERAGE=1 node tests/scripts/test-wrapper.js",
14
15
  "test:coverage:nyc": "nyc --reporter=text --reporter=lcov --reporter=html jest --config jest.config.coverage.js --runInBand",
15
16
  "test:watch": "jest --watch",
@@ -71,13 +72,16 @@
71
72
  },
72
73
  "dependencies": {
73
74
  "ajv": "^8.12.0",
75
+ "ajv-formats": "^3.0.1",
74
76
  "axios": "^1.6.0",
75
77
  "chalk": "^4.1.2",
76
78
  "commander": "^11.1.0",
77
79
  "handlebars": "^4.7.8",
78
80
  "inquirer": "^8.2.5",
81
+ "inquirer-autocomplete-prompt": "^2.0.0",
79
82
  "js-yaml": "^4.1.0",
80
- "ora": "^5.4.1"
83
+ "ora": "^5.4.1",
84
+ "yaml": "^2.4.0"
81
85
  },
82
86
  "devDependencies": {
83
87
  "@babel/preset-env": "^7.29.0",
@@ -86,6 +90,7 @@
86
90
  "cross-env": "^10.1.0",
87
91
  "eslint": "^8.55.0",
88
92
  "jest": "^30.2.0",
93
+ "markdownlint-cli": "^0.47.0",
89
94
  "nyc": "^17.1.0"
90
95
  },
91
96
  "repository": {
@@ -5,7 +5,7 @@ app:
5
5
  description: "AI Fabrix Dataplane is a secure, in-tenant integration and automation layer that supplies governed, normalized, and explainable enterprise data to AI agents. Using CIP as a declarative standard, it enforces RBAC and ABAC, executes integrations, and exposes trusted data via MCP and OpenAPI."
6
6
  type: webapp
7
7
  language: python # Explicitly specify Python language
8
- version: 1.7.0
8
+ version: 1.8.0
9
9
 
10
10
  # Image Configuration
11
11
  # Set tag to match your build (e.g. aifabrix build dataplane -t v1.0.0 then tag: v1.0.0)
@@ -1,5 +1,5 @@
1
1
  # Environment Variables Template
2
- # Use kv:// references for secrets (resolved from .aifabrix/secrets.yaml)
2
+ # Use key-value refs (format: kv://secret-key) for secrets (resolved from .aifabrix/secrets.yaml)
3
3
  # Use ${VAR} for environment-specific values
4
4
 
5
5
  # =============================================================================
@@ -8,8 +8,8 @@
8
8
 
9
9
  # HTTP port for the app
10
10
  PORT=${PORT}
11
- # development | staging | production
12
- ENVIRONMENT=development
11
+ # dev | tst | pro
12
+ ENVIRONMENT=dev
13
13
  # Enable debug mode
14
14
  DEBUG=false
15
15
  # Logging level: DEBUG, INFO, WARNING, ERROR, CRITICAL
@@ -28,7 +28,7 @@ API_KEY=kv://miso-controller-api-key-secretKeyVault
28
28
 
29
29
  # API Configuration
30
30
  API_V1_STR=/api/v1
31
- VERSION=1.7.0
31
+ VERSION=1.8.0
32
32
  # Base URL for the dataplane web server (used for default OAuth2 callback URL when redirectUri is omitted)
33
33
  DATAPLANE_WEB_SERVER_URL=kv://dataplane-web-server-url
34
34
  DATAPLANE_INTERNAL_URL=kv://dataplane-internal-server-url
@@ -105,7 +105,7 @@ KEYCLOAK_REALM=aifabrix
105
105
  # Public: browser redirects and CORS for client_token; set when controller is behind a different public URL.
106
106
  MISO_WEB_SERVER_URL=kv://miso-controller-web-server-url
107
107
  # Internal: server-to-controller API calls (auth, pipeline, status, RBAC).
108
- MISO_CONTROLLER_URL=kv://miso-controller-internal-server-url
108
+ MISO_CONTROLLER_URL=http://${MISO_HOST}:${MISO_PORT}
109
109
 
110
110
  # Pipeline env key for controller URLs: /api/v1/pipeline/{envKey}/validate and /deploy.
111
111
  # Set MISO_PIPELINE_ENV_KEY=dev when controller uses dev (e.g. MISO_CLIENTID=miso-controller-dev-dataplane).
@@ -38,8 +38,8 @@ roles:
38
38
  permissions:
39
39
  # Credential management
40
40
  - name: "credential:create"
41
- roles: ["aifabrix-platform-admin"]
42
- description: "Create credentials"
41
+ roles: ["aifabrix-platform-admin", "aifabrix-deployment-admin", "aifabrix-developer"]
42
+ description: "Create credentials (and store kv:// secrets for upload/publish)"
43
43
 
44
44
  - name: "credential:read"
45
45
  roles: ["aifabrix-platform-admin", "aifabrix-security-admin", "aifabrix-compliance-admin", "aifabrix-observer"]
@@ -59,7 +59,7 @@ ENABLE_API_DOCS=true
59
59
  # Rate Limiting Configuration (for local development)
60
60
  # Set DISABLE_RATE_LIMIT=true to disable rate limiting entirely (local development only)
61
61
  DISABLE_RATE_LIMIT=true
62
- # RATE_LIMIT_WINDOW_MS=900000 # 15 minutes in milliseconds (default: 900000)
62
+ # RATE_LIMIT_WINDOW_MS=600000 # 10 minutes in milliseconds (default: 600000)
63
63
  # RATE_LIMIT_MAX=100 # Max requests per window (default: 100)
64
64
 
65
65
  # Package Version (auto-set by npm/pnpm, optional override)
@@ -10,25 +10,29 @@
10
10
 
11
11
  ## Files
12
12
 
13
- - `application.yaml` – Application configuration with `app` and `externalIntegration` blocks
14
- - `{{systemKey}}-system.yaml` – External system definition (authentication, OpenAPI/MCP, RBAC)
13
+ - `application{{fileExt}}` – Application configuration with `app` and `externalIntegration` blocks
14
+ - `{{systemKey}}-system{{fileExt}}` – External system definition (authentication, OpenAPI/MCP, RBAC)
15
15
  {{#each datasources}}
16
16
  - `{{fileName}}` – Datasource: {{displayName}}
17
17
  {{/each}}
18
18
  - `env.template` – Environment variables template (secrets, API keys)
19
- - `{{systemKey}}-deploy.json` – Deployment manifest (generated by `aifabrix json`)
19
+ - `{{systemKey}}-deploy.json` – Deployment manifest (generated by `aifabrix json {{appName}}`)
20
+ - `deploy.js` – Deploy script for the integration
21
+ - `wizard.yaml` – Wizard configuration (if created via wizard)
20
22
 
21
23
  Optional: `rbac.yaml` – Roles and permissions merged into the system when present.
22
24
 
23
25
  ## Quick Start
24
26
 
25
- ### 1. Create External System
26
-
27
+ Login to your controller
27
28
  ```bash
28
- aifabrix create {{appName}} --type external
29
+ aifabrix auth config --set-controller <url> --set-environment dev
30
+ aifabrix login
29
31
  ```
30
32
 
31
- Or use the interactive wizard:
33
+ ### 1. Extend External System
34
+
35
+ Use the interactive wizard to extend your existing system:
32
36
 
33
37
  ```bash
34
38
  aifabrix wizard --app {{appName}}
@@ -38,36 +42,35 @@ aifabrix wizard --app {{appName}}
38
42
 
39
43
  Edit files in `integration/{{appName}}/`:
40
44
 
41
- - **Authentication**: `{{systemKey}}-system.yaml` (auth type, credentials placeholders)
42
- - **Field mappings**: `{{systemKey}}-datasource-*.yaml` (dimensions, attributes, operations)
45
+ - **Authentication**: `{{systemKey}}-system{{fileExt}}` (auth type, credentials placeholders)
46
+ - **Field mappings**: `{{systemKey}}-datasource-*-datasource{{fileExt}}` (dimensions, attributes, operations)
47
+ - **Credential and configuration**: `env.template` (security settings and configuration variables)
43
48
 
44
49
  ### 3. Validate Configuration
45
50
 
46
51
  ```bash
47
- aifabrix validate {{appName}} --type external
48
- ```
49
-
50
- ### 4. Generate Deployment Manifest
51
-
52
- ```bash
53
- aifabrix json {{appName}} --type external
52
+ aifabrix validate {{appName}}
54
53
  ```
55
54
 
56
- This creates `{{systemKey}}-deploy.json` in `integration/{{appName}}/`.
57
-
58
- ### 5. Deploy
55
+ ### 4. Repair Deployment Manifest
59
56
 
60
- Controller URL and environment are read from config. Configure and log in first:
57
+ **Run repair regularly.** It keeps naming conventions, filenames, and the deployment manifest aligned with AI Fabrix platform best practices. Use it after editing datasources, env.template, or system config—and run it often to catch drift early.
61
58
 
62
59
  ```bash
63
- aifabrix auth config --set-controller <url> --set-environment dev
64
- aifabrix login
60
+ aifabrix repair {{appName}}
65
61
  ```
66
62
 
67
- Then deploy:
63
+ Options:
64
+ --dry-run Report changes only; do not write
65
+ --rbac Ensure RBAC permissions per datasource and add default Admin/Reader roles if none exist
66
+ --expose Set exposed.attributes on each datasource to all fieldMappings.attributes keys
67
+ --sync Add default sync section to datasources that lack it
68
+ --test Generate testPayload.payloadTemplate and testPayload.expectedResult from attributes
69
+
70
+ ### 5. Upload to dataplane
68
71
 
69
72
  ```bash
70
- aifabrix deploy {{appName}}
73
+ aifabrix upload {{appName}}
71
74
  ```
72
75
 
73
76
  ## Testing
@@ -84,6 +87,43 @@ aifabrix test {{appName}}
84
87
  aifabrix test-integration {{appName}}
85
88
  ```
86
89
 
90
+ ### End-to-end Tests (Via Dataplane)
91
+
92
+ ```bash
93
+ aifabrix test-e2e {{appName}}
94
+ ```
95
+
96
+ Options:
97
+ -e, --env <env> Environment: dev, tst, or pro (builder: dev/tst for container)
98
+ -v, --verbose Show detailed step output and poll progress
99
+ --debug Include debug output and write log to integration/{{appName}}/logs/
100
+ --no-async Use sync mode (no polling); single POST per datasource
101
+
102
+ ### E2E tests per datasource
103
+
104
+ To run a full E2E test for a single datasource (config, credential, sync, data, CIP), use `aifabrix datasource test-e2e` with the datasource key and app:
105
+
106
+ {{#if hasDatasources}}
107
+ ```bash
108
+ {{#each datasources}}
109
+ # {{displayName}}
110
+ aifabrix datasource test-e2e {{datasourceKey}} --app {{../appName}}
111
+
112
+ {{/each}}
113
+ ```
114
+ {{/if}}
115
+
116
+ Options:
117
+ -a, --app {{appName}} App key (default: resolve from cwd if inside integration/{{appName}}/)
118
+ -e, --env <env> Environment: dev, tst, or pro
119
+ -v, --verbose Show detailed step output and poll progress
120
+ --debug Include debug output and write log to integration/{{appName}}/logs/
121
+ --test-crud Enable CRUD lifecycle test (body testCrud: true)
122
+ --record-id <id> Record ID for test (body recordId)
123
+ --no-cleanup Disable cleanup after test (body cleanup: false)
124
+ --primary-key-value <value|@path> Primary key value or path to JSON file (e.g. @pk.json) for body primaryKeyValue
125
+ --no-async Use sync mode (no polling); single POST, no asyncRun
126
+
87
127
  ## Deployment
88
128
 
89
129
  Deploy via miso-controller pipeline (same as regular apps). Auth and controller come from `aifabrix login` and `aifabrix auth config`:
@@ -94,6 +134,6 @@ aifabrix deploy {{appName}}
94
134
 
95
135
  ## Troubleshooting
96
136
 
97
- - **Validation errors**: Run `aifabrix validate {{appName}} --type external` to see schema and manifest errors.
137
+ - **Validation errors**: Run `aifabrix validate {{appName}}` to see schema and manifest errors.
98
138
  - **Deployment / auth**: Run `aifabrix auth config --set-controller <url> --set-environment <env>` and `aifabrix login` before `aifabrix deploy`.
99
139
  - **File not found**: Run commands from the project root (where `package.json` and `integration/` live).
@@ -10,6 +10,8 @@ const { execSync } = require('child_process');
10
10
  const path = require('path');
11
11
 
12
12
  const scriptDir = __dirname;
13
+ // Project root (repo containing integration/ and builder/) so deploy/test-integration resolve app paths correctly
14
+ const projectRoot = path.join(scriptDir, '..', '..');
13
15
  const appKey = '{{systemKey}}';
14
16
  const env = process.env.ENVIRONMENT || 'dev';
15
17
  // Controller URL: from config (aifabrix auth config) or set CONTROLLER env before running
@@ -57,12 +59,12 @@ run('aifabrix validate "' + path.join(scriptDir, '{{this}}') + '"');
57
59
  console.log('✅ Validation passed');
58
60
 
59
61
  console.log('🚀 Deploying ' + appKey + '...');
60
- run('aifabrix deploy ' + appKey);
62
+ run('aifabrix deploy ' + appKey, { cwd: projectRoot });
61
63
  console.log('✅ Deployment complete');
62
64
 
63
65
  if (process.env.RUN_TESTS !== 'false') {
64
66
  console.log('🧪 Running integration tests...');
65
- run('aifabrix test-integration ' + appKey);
67
+ run('aifabrix test-integration ' + appKey, { cwd: projectRoot });
66
68
  console.log('✅ Tests passed');
67
69
  }
68
70
 
@@ -0,0 +1,217 @@
1
+ key: "{{fullDatasourceKey}}"
2
+ displayName: "{{datasourceDisplayName}}"
3
+ description: "{{datasourceDescription}}"
4
+ systemKey: "{{systemKey}}"
5
+ entityType: "{{schemaEntityType}}"
6
+ resourceType: "{{resourceType}}"
7
+ primaryKey:
8
+ {{#each primaryKey}} - "{{this}}"
9
+ {{/each}}
10
+ enabled: true
11
+ version: "1.0.0"
12
+ fieldMappings:
13
+ {{#if dimensions}}
14
+ dimensions:
15
+ {{#each dimensions}}
16
+ {{@key}}: "{{this}}"
17
+ {{/each}}
18
+ {{else}}
19
+ dimensions: {}
20
+ # Optional: add country, department, organization for ABAC
21
+ {{/if}}
22
+ attributes:
23
+ {{#if attributes}}
24
+ {{#each attributes}}
25
+ {{@key}}:
26
+ expression: "{{this.expression}}"
27
+ type: {{this.type}}
28
+ indexed: {{#if this.indexed}}true{{else}}false{{/if}}
29
+ {{/each}}
30
+ {{else}}
31
+ id:
32
+ expression: "{{raw.id}}"
33
+ type: string
34
+ indexed: true
35
+ name:
36
+ expression: "{{raw.name}}"
37
+ type: string
38
+ indexed: false
39
+ {{/if}}
40
+ {{#if (eq systemType "openapi")}}
41
+ openapi:
42
+ enabled: true
43
+ documentKey: "{{systemKey}}-api"
44
+ operations:
45
+ list:
46
+ operationId: "list{{entityKey}}"
47
+ method: GET
48
+ path: "/{{entityKey}}"
49
+ get:
50
+ operationId: "get{{entityKey}}"
51
+ method: GET
52
+ path: "/{{entityKey}}/{id}"
53
+ autoRbac: true
54
+ {{/if}}
55
+
56
+ # --- Optional sections: uncomment or delete as needed ---
57
+ # CIP (Custom Integration Pipeline) supports: fetch, paginate, map, filter, output, pythonInline steps.
58
+ # Operations: list, get, create, update, delete. Pagination: cursor | page | offset.
59
+ {{#if (eq schemaEntityType "recordStorage")}}
60
+ # sync:
61
+ # pull:
62
+ # enabled: true
63
+ # schedule: "0 * * * *"
64
+ # capabilities: [list, get]
65
+ {{/if}}
66
+ {{#if (eq schemaEntityType "documentStorage")}}
67
+ # documentStorage:
68
+ # enabled: true
69
+ # binaryOperationRef: get
70
+ # embeddingField: content
71
+ {{/if}}
72
+ {{#if (eq schemaEntityType "vectorStore")}}
73
+ # vectorStore:
74
+ # enabled: true
75
+ # embeddingModel: "text-embedding-ada-002"
76
+ {{/if}}
77
+ {{#if (eq schemaEntityType "messageService")}}
78
+ # messageService:
79
+ # enabled: true
80
+ # channels: []
81
+ {{/if}}
82
+
83
+ # --- execution: CIP pipeline ---
84
+ # Steps: fetch (openapi/http) → paginate (cursor|page|offset) → map (useFieldMappings, inputPath) → filter (enforceAbac, expression) → output (mode: records)
85
+ # Pagination: cursor=cursorField+cursorParam | page=pageParam+pageSizeParam | offset=offsetParam+pageSizeParam
86
+ # Adjust inputPath to your API ($.results[*], $.items[*], $.value[*], etc.)
87
+ {{#if (eq schemaEntityType "recordStorage")}}
88
+ # execution:
89
+ # engine: cip
90
+ # cip:
91
+ # operations:
92
+ # list:
93
+ # enabled: true
94
+ # description: "List all records"
95
+ # steps:
96
+ # - fetch: { source: openapi, openapiRef: list, query: {} }
97
+ # - paginate: { strategy: cursor, cursorField: "$.paging.next.after", cursorParam: after, pageSize: 100, maxPages: 100 }
98
+ # # Alternative: page → pageParam, pageSizeParam | offset → offsetParam, pageSizeParam
99
+ # - map: { useFieldMappings: true, inputPath: "$.results[*]" }
100
+ # - filter: { enforceAbac: true }
101
+ # # Optional: filter.expression for SQL or JSON filter, e.g. expression: "status = 'active'"
102
+ # - output: { mode: records }
103
+ # get:
104
+ # enabled: true
105
+ # description: "Get record by ID"
106
+ # steps:
107
+ # - fetch: { source: openapi, openapiRef: get, query: {} }
108
+ # - map: { useFieldMappings: true, inputPath: "$" }
109
+ # - filter: { enforceAbac: true }
110
+ # - output: { mode: records }
111
+ # create:
112
+ # steps:
113
+ # - fetch: { source: openapi, openapiRef: create, bodyTemplate: "{{body}}" }
114
+ # - map: { useFieldMappings: true, inputPath: "$" }
115
+ # - output: { mode: records }
116
+ # update:
117
+ # steps:
118
+ # - fetch: { source: openapi, openapiRef: update, bodyTemplate: "{{body}}" }
119
+ # - map: { useFieldMappings: true, inputPath: "$" }
120
+ # - output: { mode: records }
121
+ # delete:
122
+ # steps:
123
+ # - fetch: { source: openapi, openapiRef: delete }
124
+ # - output: { mode: records }
125
+ {{/if}}
126
+ {{#if (eq schemaEntityType "documentStorage")}}
127
+ # execution:
128
+ # engine: cip
129
+ # cip:
130
+ # operations:
131
+ # list:
132
+ # enabled: true
133
+ # description: "List documents"
134
+ # steps:
135
+ # - fetch: { source: openapi, openapiRef: list, query: {} }
136
+ # - paginate: { strategy: offset, offsetParam: skip, pageSizeParam: top, pageSize: 100, maxPages: 100 }
137
+ # - map: { useFieldMappings: true, inputPath: "$.value[*]" }
138
+ # - filter: { enforceAbac: true }
139
+ # - output: { mode: records }
140
+ # get:
141
+ # enabled: true
142
+ # description: "Get document by ID"
143
+ # steps:
144
+ # - fetch: { source: openapi, openapiRef: get, query: {} }
145
+ # - map: { useFieldMappings: true, inputPath: "$" }
146
+ # - filter: { enforceAbac: true }
147
+ # - output: { mode: records }
148
+ # create:
149
+ # enabled: true
150
+ # description: "Upload document"
151
+ # steps:
152
+ # - fetch: { source: openapi, openapiRef: create, bodyTemplate: "{{fileContent}}" }
153
+ # - map: { useFieldMappings: true, inputPath: "$" }
154
+ # - output: { mode: records }
155
+ {{/if}}
156
+ {{#if (eq schemaEntityType "vectorStore")}}
157
+ # execution:
158
+ # engine: cip
159
+ # cip:
160
+ # operations:
161
+ # list:
162
+ # steps:
163
+ # - fetch: { source: openapi, openapiRef: list, query: {} }
164
+ # - map: { useFieldMappings: true, inputPath: "$" }
165
+ # - output: { mode: records }
166
+ # get:
167
+ # steps:
168
+ # - fetch: { source: openapi, openapiRef: get, query: {} }
169
+ # - map: { useFieldMappings: true, inputPath: "$" }
170
+ # - output: { mode: records }
171
+ {{/if}}
172
+ {{#if (eq schemaEntityType "messageService")}}
173
+ # execution:
174
+ # engine: cip
175
+ # cip:
176
+ # operations:
177
+ # list:
178
+ # steps:
179
+ # - fetch: { source: openapi, openapiRef: list, query: {} }
180
+ # - map: { useFieldMappings: true, inputPath: "$" }
181
+ # - output: { mode: records }
182
+ {{/if}}
183
+ {{#if (eq schemaEntityType "none")}}
184
+ # execution:
185
+ # engine: cip
186
+ # cip:
187
+ # operations:
188
+ # list:
189
+ # steps:
190
+ # - fetch: { source: openapi, openapiRef: list, query: {} }
191
+ # - map: { useFieldMappings: true, inputPath: "$" }
192
+ # - output: { mode: records }
193
+ # get:
194
+ # steps:
195
+ # - fetch: { source: openapi, openapiRef: get, query: {} }
196
+ # - map: { useFieldMappings: true, inputPath: "$" }
197
+ # - output: { mode: records }
198
+ {{/if}}
199
+
200
+ # config: ABAC cross-system filters (SQL or JSON)
201
+ # config:
202
+ # abac:
203
+ # crossSystemSql: "country = '{{actor.country}}'"
204
+ # # crossSystemJson: { "dimensions.country": { "eq": "{{actor.country}}" } }
205
+
206
+ # capabilities: [list, get, create, update, delete]
207
+
208
+ # exposed: attributes to expose via MCP/OpenAPI
209
+ # exposed:
210
+ # attributes: [id, name]
211
+
212
+ # metadataSchema: JSON Schema for raw metadata validation
213
+ # metadataSchema:
214
+ # type: object
215
+ # properties:
216
+ # id: { type: string }
217
+ # name: { type: string }