@aifabrix/builder 2.41.0 → 2.42.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/.cursor/rules/docs-rules.mdc +30 -0
  2. package/README.md +2 -2
  3. package/integration/hubspot/README.md +11 -5
  4. package/integration/hubspot/application.json +54 -0
  5. package/integration/hubspot/create-hubspot.js +9 -136
  6. package/integration/hubspot/env.template +3 -4
  7. package/integration/hubspot/hubspot-datasource-company.json +343 -5
  8. package/integration/hubspot/hubspot-datasource-contact.json +413 -5
  9. package/integration/hubspot/hubspot-datasource-deal.json +341 -4
  10. package/integration/hubspot/hubspot-datasource-users.json +116 -0
  11. package/integration/hubspot/hubspot-deploy.json +1250 -108
  12. package/integration/hubspot/hubspot-system.json +15 -32
  13. package/integration/hubspot/test-dataplane-down-tests.js +17 -16
  14. package/integration/hubspot/test-dataplane-down.js +2 -2
  15. package/jest.config.manual.js +2 -1
  16. package/lib/api/external-test.api.js +111 -0
  17. package/lib/api/index.js +42 -19
  18. package/lib/api/pipeline.api.js +66 -120
  19. package/lib/api/types/pipeline.types.js +37 -0
  20. package/lib/api/wizard-platform.api.js +61 -0
  21. package/lib/api/wizard.api.js +36 -2
  22. package/lib/app/config.js +23 -11
  23. package/lib/app/index.js +5 -3
  24. package/lib/app/prompts.js +46 -31
  25. package/lib/app/readme.js +11 -4
  26. package/lib/app/run-env-compose.js +64 -1
  27. package/lib/app/run-helpers.js +1 -1
  28. package/lib/app/show-display.js +1 -1
  29. package/lib/cli/setup-app.js +45 -14
  30. package/lib/cli/setup-credential-deployment.js +31 -6
  31. package/lib/cli/setup-dev.js +27 -0
  32. package/lib/cli/setup-environment.js +12 -4
  33. package/lib/cli/setup-external-system.js +19 -4
  34. package/lib/cli/setup-infra.js +54 -14
  35. package/lib/cli/setup-utility.js +117 -21
  36. package/lib/commands/auth-config.js +22 -12
  37. package/lib/commands/credential-env.js +162 -0
  38. package/lib/commands/credential-list.js +17 -22
  39. package/lib/commands/credential-push.js +96 -0
  40. package/lib/commands/datasource.js +77 -6
  41. package/lib/commands/dev-init.js +39 -1
  42. package/lib/commands/repair-auth-config.js +99 -0
  43. package/lib/commands/repair-datasource-keys.js +208 -0
  44. package/lib/commands/repair-datasource.js +235 -0
  45. package/lib/commands/repair-env-template.js +348 -0
  46. package/lib/commands/repair-internal.js +85 -0
  47. package/lib/commands/repair-rbac.js +158 -0
  48. package/lib/commands/repair.js +518 -0
  49. package/lib/commands/secrets-set.js +6 -0
  50. package/lib/commands/test-e2e-external.js +165 -0
  51. package/lib/commands/up-dataplane.js +90 -6
  52. package/lib/commands/upload.js +71 -40
  53. package/lib/commands/wizard-core-helpers.js +230 -5
  54. package/lib/commands/wizard-core.js +68 -29
  55. package/lib/commands/wizard-dataplane.js +1 -1
  56. package/lib/commands/wizard-entity-selection.js +43 -0
  57. package/lib/commands/wizard-headless.js +49 -5
  58. package/lib/commands/wizard-helpers.js +7 -3
  59. package/lib/commands/wizard.js +93 -64
  60. package/lib/core/config.js +7 -1
  61. package/lib/core/secrets.js +33 -12
  62. package/lib/datasource/deploy.js +12 -3
  63. package/lib/datasource/test-e2e.js +219 -0
  64. package/lib/datasource/test-integration.js +154 -0
  65. package/lib/deployment/deployer.js +7 -5
  66. package/lib/external-system/download-helpers.js +3 -1
  67. package/lib/external-system/download.js +182 -204
  68. package/lib/external-system/generator.js +204 -56
  69. package/lib/external-system/test-execution.js +2 -1
  70. package/lib/external-system/test-system-level.js +73 -0
  71. package/lib/external-system/test.js +51 -18
  72. package/lib/generator/external-controller-manifest.js +29 -2
  73. package/lib/generator/external-schema-utils.js +4 -2
  74. package/lib/generator/external.js +10 -3
  75. package/lib/generator/index.js +4 -1
  76. package/lib/generator/split-readme.js +1 -0
  77. package/lib/generator/split-variables.js +7 -1
  78. package/lib/generator/split.js +194 -54
  79. package/lib/generator/wizard-prompts-secondary.js +326 -0
  80. package/lib/generator/wizard-prompts.js +105 -106
  81. package/lib/generator/wizard-readme.js +91 -0
  82. package/lib/generator/wizard.js +180 -179
  83. package/lib/infrastructure/compose.js +11 -1
  84. package/lib/infrastructure/index.js +11 -3
  85. package/lib/infrastructure/services.js +22 -11
  86. package/lib/schema/application-schema.json +8 -5
  87. package/lib/schema/external-datasource.schema.json +49 -26
  88. package/lib/schema/external-system.schema.json +82 -6
  89. package/lib/schema/wizard-config.schema.json +23 -1
  90. package/lib/utils/api.js +38 -10
  91. package/lib/utils/auth-headers.js +8 -7
  92. package/lib/utils/compose-generator.js +1 -1
  93. package/lib/utils/compose-handlebars-helpers.js +11 -0
  94. package/lib/utils/config-format-preference.js +51 -0
  95. package/lib/utils/config-format.js +36 -0
  96. package/lib/utils/configuration-env-resolver.js +179 -0
  97. package/lib/utils/credential-display.js +83 -0
  98. package/lib/utils/credential-secrets-env.js +115 -25
  99. package/lib/utils/dataplane-pipeline-warning.js +28 -0
  100. package/lib/utils/deployment-validation-helpers.js +4 -4
  101. package/lib/utils/dev-ca-install.js +139 -0
  102. package/lib/utils/env-copy.js +23 -3
  103. package/lib/utils/error-formatters/http-status-errors.js +0 -1
  104. package/lib/utils/error-formatters/permission-errors.js +0 -1
  105. package/lib/utils/error-formatters/validation-errors.js +0 -1
  106. package/lib/utils/external-readme.js +89 -30
  107. package/lib/utils/external-system-display.js +59 -1
  108. package/lib/utils/external-system-test-helpers.js +21 -8
  109. package/lib/utils/external-system-validators.js +3 -0
  110. package/lib/utils/file-upload.js +20 -50
  111. package/lib/utils/help-builder.js +1 -0
  112. package/lib/utils/infra-status.js +50 -44
  113. package/lib/utils/local-secrets.js +5 -5
  114. package/lib/utils/paths.js +85 -4
  115. package/lib/utils/secrets-canonical.js +93 -0
  116. package/lib/utils/secrets-generator.js +20 -0
  117. package/lib/utils/secrets-helpers.js +75 -89
  118. package/lib/utils/test-log-writer.js +56 -0
  119. package/lib/utils/token-manager.js +24 -32
  120. package/lib/validation/env-template-auth.js +157 -0
  121. package/lib/validation/env-template-kv.js +41 -0
  122. package/lib/validation/external-manifest-validator.js +25 -0
  123. package/lib/validation/external-system-auth-rules.js +86 -0
  124. package/lib/validation/validate-batch.js +149 -0
  125. package/lib/validation/validate-datasource-keys-api.js +33 -0
  126. package/lib/validation/validate-display.js +94 -16
  127. package/lib/validation/validate.js +25 -12
  128. package/lib/validation/validator.js +7 -9
  129. package/lib/validation/wizard-datasource-validation.js +50 -0
  130. package/package.json +7 -2
  131. package/templates/applications/dataplane/application.yaml +1 -1
  132. package/templates/applications/dataplane/env.template +5 -5
  133. package/templates/applications/dataplane/rbac.yaml +2 -2
  134. package/templates/applications/miso-controller/env.template +1 -1
  135. package/templates/external-system/README.md.hbs +75 -22
  136. package/templates/external-system/deploy.js.hbs +4 -2
  137. package/templates/external-system/external-datasource.yaml.hbs +217 -0
  138. package/templates/external-system/external-system.json.hbs +1 -18
  139. package/templates/infra/compose.yaml.hbs +6 -0
  140. package/templates/python/docker-compose.hbs +4 -4
  141. package/templates/typescript/docker-compose.hbs +4 -4
  142. package/integration/hubspot/application.yaml +0 -37
@@ -8,6 +8,7 @@
8
8
  * @version 2.0.0
9
9
  */
10
10
 
11
+ const path = require('path');
11
12
  const chalk = require('chalk');
12
13
  const logger = require('../utils/logger');
13
14
  const { loadConfigFile } = require('../utils/config-format');
@@ -242,22 +243,25 @@ function displayApplicationStep(application) {
242
243
  * @returns {void}
243
244
  */
244
245
  function displayComponentsStep(components) {
245
- if (!components.files || components.files.length === 0) {
246
+ const hasFiles = components.files && components.files.length > 0;
247
+ const hasErrors = components.errors && components.errors.length > 0;
248
+ if (!hasFiles && !hasErrors) {
246
249
  return;
247
250
  }
248
251
 
249
252
  logger.log(chalk.blue('\nExternal Integration Files:'));
250
- components.files.forEach(file => {
251
- if (file.valid) {
252
- logger.log(chalk.green(` ✓ ${file.file} (${file.type})`));
253
- } else {
254
- logger.log(chalk.red(` ✗ ${file.file} (${file.type})`));
255
- }
256
- });
257
-
258
- if (components.errors && components.errors.length > 0) {
253
+ if (hasFiles) {
254
+ components.files.forEach(file => {
255
+ if (file.valid) {
256
+ logger.log(chalk.green(` ✓ ${file.file} (${file.type})`));
257
+ } else {
258
+ logger.log(chalk.red(` ✗ ${file.file} (${file.type})`));
259
+ }
260
+ });
261
+ }
262
+ if (hasErrors) {
259
263
  components.errors.forEach(error => {
260
- logger.log(chalk.red(` • ${error}`));
264
+ logger.log(chalk.red(` • ${error}`));
261
265
  });
262
266
  }
263
267
  }
@@ -301,7 +305,9 @@ function displayDimensionsStep(datasourceFiles) {
301
305
  */
302
306
  function displayManifestStep(manifest, componentFiles) {
303
307
  logger.log(chalk.blue('\nDeployment Manifest:'));
304
- if (manifest.valid) {
308
+ if (manifest.skipped) {
309
+ logger.log(chalk.yellow(' ⊘ Skipped (fix errors above first)'));
310
+ } else if (manifest.valid) {
305
311
  logger.log(chalk.green(' ✓ Full deployment manifest is valid'));
306
312
  if (componentFiles) {
307
313
  const datasourceFiles = componentFiles.filter(f => f.type === 'datasource' || f.type === 'external-datasource');
@@ -311,10 +317,14 @@ function displayManifestStep(manifest, componentFiles) {
311
317
  }
312
318
  } else {
313
319
  logger.log(chalk.red(' ✗ Full deployment manifest validation failed:'));
314
- if (manifest.errors && manifest.errors.length > 0) {
315
- manifest.errors.forEach(error => {
316
- logger.log(chalk.red(` • ${error}`));
320
+ const errs = manifest.errors && manifest.errors.length > 0 ? manifest.errors : [];
321
+ if (errs.length > 0) {
322
+ errs.forEach(error => {
323
+ const msg = typeof error === 'string' ? error : String(error);
324
+ logger.log(chalk.red(` • ${msg}`));
317
325
  });
326
+ } else {
327
+ logger.log(chalk.red(' • No error details available (check schema and manifest structure).'));
318
328
  }
319
329
  }
320
330
  if (manifest.warnings && manifest.warnings.length > 0) {
@@ -338,7 +348,7 @@ function displayStepByStepValidation(result) {
338
348
 
339
349
  displayApplicationStep(result.steps.application);
340
350
 
341
- if (result.steps.components.files && result.steps.components.files.length > 0) {
351
+ if (!result.steps.components.valid || (result.steps.components.files && result.steps.components.files.length > 0)) {
342
352
  displayComponentsStep(result.steps.components);
343
353
  }
344
354
 
@@ -356,6 +366,68 @@ function displayStepByStepValidation(result) {
356
366
  if (result.warnings && result.warnings.length > 0) {
357
367
  displayAggregatedWarnings(result.warnings);
358
368
  }
369
+
370
+ displayOverallStatus(result);
371
+ if (result.appPath) {
372
+ logger.log(chalk.gray(`\n${path.resolve(result.appPath)}`));
373
+ }
374
+ }
375
+
376
+ /**
377
+ * Displays batch validation results (per-app blocks then summary).
378
+ * Expects batchResult.batch === true and batchResult.results.
379
+ * @function displayBatchValidationResults
380
+ * @param {Object} batchResult - Batch result from validateAllIntegrations / validateAllBuilderApps / validateAll
381
+ */
382
+ function displayBatchValidationResults(batchResult) {
383
+ if (!batchResult || batchResult.batch !== true || !Array.isArray(batchResult.results)) {
384
+ return;
385
+ }
386
+
387
+ const results = batchResult.results;
388
+ results.forEach(item => {
389
+ logger.log(chalk.blue(`\n--- ${item.appName} ---`));
390
+ if (item.error) {
391
+ logger.log(chalk.red(` ✗ ${item.error}`));
392
+ } else if (item.result) {
393
+ displayValidationResults(item.result);
394
+ }
395
+ });
396
+
397
+ const passed = results.filter(r => r.result && r.result.valid).length;
398
+ const failed = results.length - passed;
399
+ logger.log(chalk.blue('\n--- Summary ---'));
400
+ if (failed === 0) {
401
+ logger.log(chalk.green(`✓ ${passed} passed, 0 failed`));
402
+ logger.log(chalk.green('Overall: Passed'));
403
+ } else {
404
+ logger.log(chalk.red(`✗ ${passed} passed, ${failed} failed`));
405
+ logger.log(chalk.red('Overall: Failed'));
406
+ }
407
+ }
408
+
409
+ /**
410
+ * Displays overall validation status
411
+ * @param {Object} result - Validation result
412
+ */
413
+ function displayOverallStatus(result) {
414
+ let hasErrors = result.errors && result.errors.length > 0;
415
+ if (!hasErrors && result.steps) {
416
+ const stepErrors = [result.steps.application, result.steps.components, result.steps.manifest]
417
+ .filter(Boolean)
418
+ .some(s => s.errors && s.errors.length > 0);
419
+ if (stepErrors) {
420
+ hasErrors = true;
421
+ }
422
+ }
423
+ const hasWarnings = result.warnings && result.warnings.length > 0;
424
+ if (hasErrors) {
425
+ logger.log(chalk.red('\nOverall: Failed'));
426
+ } else if (hasWarnings) {
427
+ logger.log(chalk.yellow('\nOverall: Passed with warnings'));
428
+ } else {
429
+ logger.log(chalk.green('\nOverall: Passed'));
430
+ }
359
431
  }
360
432
 
361
433
  /**
@@ -390,10 +462,16 @@ function displayValidationResults(result) {
390
462
  // Combine all warnings
391
463
  const allWarnings = [...(result.warnings || []), ...dimensionWarnings];
392
464
  displayAggregatedWarnings(allWarnings);
465
+
466
+ displayOverallStatus(result);
467
+ if (result.appPath) {
468
+ logger.log(chalk.gray(`\n${path.resolve(result.appPath)}`));
469
+ }
393
470
  }
394
471
 
395
472
  module.exports = {
396
473
  displayValidationResults,
474
+ displayBatchValidationResults,
397
475
  displayStepByStepValidation,
398
476
  displayApplicationValidation,
399
477
  displayExternalFilesValidation,
@@ -16,11 +16,16 @@ const { resolveExternalFiles } = require('../utils/schema-resolver');
16
16
  const { loadExternalSystemSchema, loadExternalDataSourceSchema, detectSchemaType } = require('../utils/schema-loader');
17
17
  const { formatValidationErrors } = require('../utils/error-formatter');
18
18
  const { detectAppType } = require('../utils/paths');
19
+ const batch = require('./validate-batch');
19
20
  const { logOfflinePathWhenType } = require('../utils/cli-utils');
20
21
  const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
21
- const { displayValidationResults } = require('./validate-display');
22
+ const { displayValidationResults, displayBatchValidationResults } = require('./validate-display');
22
23
  const { generateControllerManifest } = require('../generator/external-controller-manifest');
23
24
  const { validateControllerManifest } = require('./external-manifest-validator');
25
+ const {
26
+ validateOAuth2GrantTypeAndAuthorizationUrl,
27
+ validateConfigurationNoStandardAuthVariables
28
+ } = require('./external-system-auth-rules');
24
29
 
25
30
  /**
26
31
  * Validates a file path (detects type and validates)
@@ -191,6 +196,8 @@ async function validateExternalFile(filePath, type) {
191
196
 
192
197
  if (normalizedType === 'system') {
193
198
  validateRoleReferences(parseResult.parsed, errors);
199
+ validateOAuth2GrantTypeAndAuthorizationUrl(parseResult.parsed, errors);
200
+ validateConfigurationNoStandardAuthVariables(parseResult.parsed, errors);
194
201
  }
195
202
 
196
203
  return {
@@ -398,6 +405,7 @@ async function validateExternalSystemComplete(appName, options = {}) {
398
405
  throw new Error('App name is required and must be a string');
399
406
  }
400
407
 
408
+ const { appPath } = await detectAppType(appName, options);
401
409
  const steps = {
402
410
  application: { valid: false, errors: [], warnings: [] },
403
411
  components: { valid: false, errors: [], warnings: [], files: [] },
@@ -414,11 +422,13 @@ async function validateExternalSystemComplete(appName, options = {}) {
414
422
 
415
423
  // If components have errors, return early (don't validate manifest)
416
424
  if (!steps.components.valid) {
425
+ steps.manifest = { valid: true, errors: [], warnings: [], skipped: true };
417
426
  return {
418
427
  valid: false,
419
428
  errors: [...steps.application.errors, ...steps.components.errors],
420
429
  warnings: [...steps.application.warnings, ...steps.components.warnings],
421
- steps
430
+ steps,
431
+ appPath
422
432
  };
423
433
  }
424
434
 
@@ -433,7 +443,8 @@ async function validateExternalSystemComplete(appName, options = {}) {
433
443
  valid: allErrors.length === 0,
434
444
  errors: allErrors,
435
445
  warnings: allWarnings,
436
- steps
446
+ steps,
447
+ appPath
437
448
  };
438
449
  }
439
450
 
@@ -451,29 +462,31 @@ async function validateAppOrFile(appOrFile, options = {}) {
451
462
  const { appPath, isExternal } = await detectAppType(appName);
452
463
  logOfflinePathWhenType(appPath);
453
464
 
454
- if (isExternal) {
455
- return await validateExternalSystemComplete(appName, options);
456
- }
465
+ if (isExternal) return await validateExternalSystemComplete(appName, options);
457
466
 
458
467
  const appValidation = await validator.validateApplication(appName, options);
459
468
  const rbacValidation = await validateRbacForExternalSystem(isExternal, appName);
460
-
461
469
  const variablesPath = resolveApplicationConfigPath(appPath);
462
470
  const earlyReturn = loadVariablesAndCheckExternalIntegration(variablesPath, appValidation);
463
- if (earlyReturn) {
464
- return earlyReturn;
465
- }
471
+ if (earlyReturn) return earlyReturn;
466
472
 
467
473
  const externalValidations = await validateExternalFilesForApp(appName, options);
468
- return aggregateValidationResults(appValidation, externalValidations, rbacValidation);
474
+ const result = aggregateValidationResults(appValidation, externalValidations, rbacValidation);
475
+ result.appPath = appPath;
476
+ return result;
469
477
  }
470
478
 
471
479
  module.exports = {
472
480
  validateAppOrFile,
473
481
  validateExternalSystemComplete,
474
482
  displayValidationResults,
483
+ displayBatchValidationResults,
475
484
  validateExternalFile,
476
485
  validateExternalFilesForApp,
477
- validateFilePath
486
+ validateFilePath,
487
+ validateAllIntegrations: (opts = {}) => batch.validateAllIntegrations(validateAppOrFile, opts),
488
+ validateAllBuilderApps: (opts = {}) => batch.validateAllBuilderApps(validateAppOrFile, opts),
489
+ validateAll: (opts = {}) => batch.validateAll(validateAppOrFile, opts),
490
+ buildBatchResult: batch.buildBatchResult
478
491
  };
479
492
 
@@ -21,6 +21,8 @@ const { checkEnvironment } = require('../utils/environment-checker');
21
21
  const { formatValidationErrors } = require('../utils/error-formatter');
22
22
  const { detectAppType, resolveApplicationConfigPath } = require('../utils/paths');
23
23
  const { loadConfigFile } = require('../utils/config-format');
24
+ const { validateAuthKvCoverage } = require('./env-template-auth');
25
+ const { validateKvReferencesInLines } = require('./env-template-kv');
24
26
 
25
27
  /**
26
28
  * Validates application config file against application schema
@@ -250,7 +252,7 @@ async function validateEnvTemplate(appName, options = {}) {
250
252
  }
251
253
 
252
254
  // Support both builder/ and integration/ directories using detectAppType
253
- const { appPath } = await detectAppType(appName, options);
255
+ const { appPath, isExternal } = await detectAppType(appName, options);
254
256
  const templatePath = path.join(appPath, 'env.template');
255
257
 
256
258
  if (!fs.existsSync(templatePath)) {
@@ -281,14 +283,10 @@ async function validateEnvTemplate(appName, options = {}) {
281
283
  }
282
284
  });
283
285
 
284
- // Check for kv:// reference format
285
- const kvPattern = /kv:\/\/([a-zA-Z0-9-_]+)/g;
286
- let match;
287
- while ((match = kvPattern.exec(content)) !== null) {
288
- const secretKey = match[1];
289
- if (!secretKey) {
290
- errors.push('Invalid kv:// reference format');
291
- }
286
+ validateKvReferencesInLines(lines, errors);
287
+
288
+ if (isExternal) {
289
+ await validateAuthKvCoverage(appPath, content, errors, warnings, options);
292
290
  }
293
291
 
294
292
  return {
@@ -0,0 +1,50 @@
1
+ /**
2
+ * @fileoverview Wizard datasource validation helpers - validate datasourceKeys and entityName against dataplane
3
+ * @author AI Fabrix Team
4
+ * @version 2.0.0
5
+ */
6
+
7
+ /**
8
+ * Validate that all datasourceKeys exist in the platform's available datasources
9
+ * @function validateDatasourceKeysForPlatform
10
+ * @param {string[]} datasourceKeys - User-provided datasource keys
11
+ * @param {Array<{key: string, displayName?: string, entity?: string}>} availableDatasources - Platform datasources
12
+ * @returns {{ valid: boolean, invalidKeys: string[] }} Validation result
13
+ */
14
+ function validateDatasourceKeysForPlatform(datasourceKeys, availableDatasources) {
15
+ if (!Array.isArray(datasourceKeys) || datasourceKeys.length === 0) {
16
+ return { valid: true, invalidKeys: [] };
17
+ }
18
+ const availableKeys = Array.isArray(availableDatasources)
19
+ ? availableDatasources.map(d => (d && typeof d === 'object' && d.key) ? d.key : null).filter(Boolean)
20
+ : [];
21
+ const invalidKeys = datasourceKeys.filter(k => !availableKeys.includes(k));
22
+ return {
23
+ valid: invalidKeys.length === 0,
24
+ invalidKeys
25
+ };
26
+ }
27
+
28
+ /**
29
+ * Validate that entityName exists in the discovered entities list
30
+ * @function validateEntityNameForOpenApi
31
+ * @param {string} entityName - User-provided entity name
32
+ * @param {Array<{name: string}>} entities - Discovered entities from OpenAPI
33
+ * @returns {{ valid: boolean }} Validation result
34
+ */
35
+ function validateEntityNameForOpenApi(entityName, entities) {
36
+ if (!entityName || typeof entityName !== 'string' || entityName.trim() === '') {
37
+ return { valid: true };
38
+ }
39
+ const entityNames = Array.isArray(entities)
40
+ ? entities.map(e => (e && typeof e === 'object' && e.name) ? e.name : null).filter(Boolean)
41
+ : [];
42
+ return {
43
+ valid: entityNames.includes(entityName)
44
+ };
45
+ }
46
+
47
+ module.exports = {
48
+ validateDatasourceKeysForPlatform,
49
+ validateEntityNameForOpenApi
50
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@aifabrix/builder",
3
- "version": "2.41.0",
3
+ "version": "2.42.1",
4
4
  "description": "AI Fabrix Local Fabric & Deployment SDK",
5
5
  "main": "lib/index.js",
6
6
  "bin": {
@@ -10,6 +10,7 @@
10
10
  "scripts": {
11
11
  "test": "node tests/scripts/test-wrapper.js",
12
12
  "test:ci": "bash tests/scripts/ci-simulate.sh",
13
+ "test:same-as-github": "npm run build:ci",
13
14
  "test:coverage": "cross-env RUN_COVERAGE=1 node tests/scripts/test-wrapper.js",
14
15
  "test:coverage:nyc": "nyc --reporter=text --reporter=lcov --reporter=html jest --config jest.config.coverage.js --runInBand",
15
16
  "test:watch": "jest --watch",
@@ -71,13 +72,16 @@
71
72
  },
72
73
  "dependencies": {
73
74
  "ajv": "^8.12.0",
75
+ "ajv-formats": "^3.0.1",
74
76
  "axios": "^1.6.0",
75
77
  "chalk": "^4.1.2",
76
78
  "commander": "^11.1.0",
77
79
  "handlebars": "^4.7.8",
78
80
  "inquirer": "^8.2.5",
81
+ "inquirer-autocomplete-prompt": "^2.0.0",
79
82
  "js-yaml": "^4.1.0",
80
- "ora": "^5.4.1"
83
+ "ora": "^5.4.1",
84
+ "yaml": "^2.4.0"
81
85
  },
82
86
  "devDependencies": {
83
87
  "@babel/preset-env": "^7.29.0",
@@ -86,6 +90,7 @@
86
90
  "cross-env": "^10.1.0",
87
91
  "eslint": "^8.55.0",
88
92
  "jest": "^30.2.0",
93
+ "markdownlint-cli": "^0.47.0",
89
94
  "nyc": "^17.1.0"
90
95
  },
91
96
  "repository": {
@@ -5,7 +5,7 @@ app:
5
5
  description: "AI Fabrix Dataplane is a secure, in-tenant integration and automation layer that supplies governed, normalized, and explainable enterprise data to AI agents. Using CIP as a declarative standard, it enforces RBAC and ABAC, executes integrations, and exposes trusted data via MCP and OpenAPI."
6
6
  type: webapp
7
7
  language: python # Explicitly specify Python language
8
- version: 1.7.0
8
+ version: 1.8.0
9
9
 
10
10
  # Image Configuration
11
11
  # Set tag to match your build (e.g. aifabrix build dataplane -t v1.0.0 then tag: v1.0.0)
@@ -1,5 +1,5 @@
1
1
  # Environment Variables Template
2
- # Use kv:// references for secrets (resolved from .aifabrix/secrets.yaml)
2
+ # Use key-value refs (format: kv://secret-key) for secrets (resolved from .aifabrix/secrets.yaml)
3
3
  # Use ${VAR} for environment-specific values
4
4
 
5
5
  # =============================================================================
@@ -8,8 +8,8 @@
8
8
 
9
9
  # HTTP port for the app
10
10
  PORT=${PORT}
11
- # development | staging | production
12
- ENVIRONMENT=development
11
+ # dev | tst | pro
12
+ ENVIRONMENT=dev
13
13
  # Enable debug mode
14
14
  DEBUG=false
15
15
  # Logging level: DEBUG, INFO, WARNING, ERROR, CRITICAL
@@ -28,7 +28,7 @@ API_KEY=kv://miso-controller-api-key-secretKeyVault
28
28
 
29
29
  # API Configuration
30
30
  API_V1_STR=/api/v1
31
- VERSION=1.7.0
31
+ VERSION=1.8.0
32
32
  # Base URL for the dataplane web server (used for default OAuth2 callback URL when redirectUri is omitted)
33
33
  DATAPLANE_WEB_SERVER_URL=kv://dataplane-web-server-url
34
34
  DATAPLANE_INTERNAL_URL=kv://dataplane-internal-server-url
@@ -105,7 +105,7 @@ KEYCLOAK_REALM=aifabrix
105
105
  # Public: browser redirects and CORS for client_token; set when controller is behind a different public URL.
106
106
  MISO_WEB_SERVER_URL=kv://miso-controller-web-server-url
107
107
  # Internal: server-to-controller API calls (auth, pipeline, status, RBAC).
108
- MISO_CONTROLLER_URL=kv://miso-controller-internal-server-url
108
+ MISO_CONTROLLER_URL=http://${MISO_HOST}:${MISO_PORT}
109
109
 
110
110
  # Pipeline env key for controller URLs: /api/v1/pipeline/{envKey}/validate and /deploy.
111
111
  # Set MISO_PIPELINE_ENV_KEY=dev when controller uses dev (e.g. MISO_CLIENTID=miso-controller-dev-dataplane).
@@ -38,8 +38,8 @@ roles:
38
38
  permissions:
39
39
  # Credential management
40
40
  - name: "credential:create"
41
- roles: ["aifabrix-platform-admin"]
42
- description: "Create credentials"
41
+ roles: ["aifabrix-platform-admin", "aifabrix-deployment-admin", "aifabrix-developer"]
42
+ description: "Create credentials (and store kv:// secrets for upload/publish)"
43
43
 
44
44
  - name: "credential:read"
45
45
  roles: ["aifabrix-platform-admin", "aifabrix-security-admin", "aifabrix-compliance-admin", "aifabrix-observer"]
@@ -59,7 +59,7 @@ ENABLE_API_DOCS=true
59
59
  # Rate Limiting Configuration (for local development)
60
60
  # Set DISABLE_RATE_LIMIT=true to disable rate limiting entirely (local development only)
61
61
  DISABLE_RATE_LIMIT=true
62
- # RATE_LIMIT_WINDOW_MS=900000 # 15 minutes in milliseconds (default: 900000)
62
+ # RATE_LIMIT_WINDOW_MS=600000 # 10 minutes in milliseconds (default: 600000)
63
63
  # RATE_LIMIT_MAX=100 # Max requests per window (default: 100)
64
64
 
65
65
  # Package Version (auto-set by npm/pnpm, optional override)
@@ -10,25 +10,29 @@
10
10
 
11
11
  ## Files
12
12
 
13
- - `application.yaml` – Application configuration with `app` and `externalIntegration` blocks
14
- - `{{systemKey}}-system.yaml` – External system definition (authentication, OpenAPI/MCP, RBAC)
13
+ - `application{{fileExt}}` – Application configuration with `app` and `externalIntegration` blocks
14
+ - `{{systemKey}}-system{{fileExt}}` – External system definition (authentication, OpenAPI/MCP, RBAC)
15
15
  {{#each datasources}}
16
16
  - `{{fileName}}` – Datasource: {{displayName}}
17
17
  {{/each}}
18
18
  - `env.template` – Environment variables template (secrets, API keys)
19
- - `{{systemKey}}-deploy.json` – Deployment manifest (generated by `aifabrix json`)
19
+ - `{{systemKey}}-deploy.json` – Deployment manifest (generated by `aifabrix json {{appName}}`)
20
+ - `deploy.js` – Deploy script for the integration
21
+ - `wizard.yaml` – Wizard configuration (if created via wizard)
20
22
 
21
23
  Optional: `rbac.yaml` – Roles and permissions merged into the system when present.
22
24
 
23
25
  ## Quick Start
24
26
 
25
- ### 1. Create External System
26
-
27
+ Login to your controller
27
28
  ```bash
28
- aifabrix create {{appName}} --type external
29
+ aifabrix auth config --set-controller <url> --set-environment dev
30
+ aifabrix login
29
31
  ```
30
32
 
31
- Or use the interactive wizard:
33
+ ### 1. Extend External System
34
+
35
+ Use the interactive wizard to extend your existing system:
32
36
 
33
37
  ```bash
34
38
  aifabrix wizard --app {{appName}}
@@ -38,36 +42,48 @@ aifabrix wizard --app {{appName}}
38
42
 
39
43
  Edit files in `integration/{{appName}}/`:
40
44
 
41
- - **Authentication**: `{{systemKey}}-system.yaml` (auth type, credentials placeholders)
42
- - **Field mappings**: `{{systemKey}}-datasource-*.yaml` (dimensions, attributes, operations)
45
+ - **Authentication**: `{{systemKey}}-system{{fileExt}}` (auth type, credentials placeholders)
46
+ - **Field mappings**: `{{systemKey}}-datasource-*-datasource{{fileExt}}` (dimensions, attributes, operations)
47
+ - **Credential and configuration**: `env.template` (security settings and configuration variables)
43
48
 
44
- ### 3. Validate Configuration
49
+ {{#if secretPaths}}{{#if secretPaths.length}}
50
+ ### Secrets
51
+
52
+ Secrets are resolved from `.aifabrix` or key vault. Set them with:
45
53
 
46
54
  ```bash
47
- aifabrix validate {{appName}} --type external
55
+ {{#each secretPaths}}
56
+ aifabrix secret set {{path}} <your value> # {{description}}
57
+ {{/each}}
48
58
  ```
59
+ {{/if}}{{/if}}
49
60
 
50
- ### 4. Generate Deployment Manifest
61
+ ### 3. Validate Configuration
51
62
 
52
63
  ```bash
53
- aifabrix json {{appName}} --type external
64
+ aifabrix validate {{appName}}
54
65
  ```
55
66
 
56
- This creates `{{systemKey}}-deploy.json` in `integration/{{appName}}/`.
57
-
58
- ### 5. Deploy
67
+ ### 4. Repair Deployment Manifest
59
68
 
60
- Controller URL and environment are read from config. Configure and log in first:
69
+ **Run repair regularly.** It keeps naming conventions, filenames, and the deployment manifest aligned with AI Fabrix platform best practices. Use it after editing datasources, env.template, or system config—and run it often to catch drift early.
61
70
 
62
71
  ```bash
63
- aifabrix auth config --set-controller <url> --set-environment dev
64
- aifabrix login
72
+ aifabrix repair {{appName}}
65
73
  ```
66
74
 
67
- Then deploy:
75
+ Options:
76
+ --auth <method> Set authentication method (oauth2, aad, apikey, basic, queryParam, oidc, hmac, none); updates system file and env.template
77
+ --dry-run Report changes only; do not write
78
+ --rbac Ensure RBAC permissions per datasource and add default Admin/Reader roles if none exist
79
+ --expose Set exposed.attributes on each datasource to all fieldMappings.attributes keys
80
+ --sync Add default sync section to datasources that lack it
81
+ --test Generate testPayload.payloadTemplate and testPayload.expectedResult from attributes
82
+
83
+ ### 5. Upload to dataplane
68
84
 
69
85
  ```bash
70
- aifabrix deploy {{appName}}
86
+ aifabrix upload {{appName}}
71
87
  ```
72
88
 
73
89
  ## Testing
@@ -84,6 +100,43 @@ aifabrix test {{appName}}
84
100
  aifabrix test-integration {{appName}}
85
101
  ```
86
102
 
103
+ ### End-to-end Tests (Via Dataplane)
104
+
105
+ ```bash
106
+ aifabrix test-e2e {{appName}}
107
+ ```
108
+
109
+ Options:
110
+ -e, --env <env> Environment: dev, tst, or pro (builder: dev/tst for container)
111
+ -v, --verbose Show detailed step output and poll progress
112
+ --debug Include debug output and write log to integration/{{appName}}/logs/
113
+ --no-async Use sync mode (no polling); single POST per datasource
114
+
115
+ ### E2E tests per datasource
116
+
117
+ To run a full E2E test for a single datasource (config, credential, sync, data, CIP), use `aifabrix datasource test-e2e` with the datasource key and app:
118
+
119
+ {{#if hasDatasources}}
120
+ ```bash
121
+ {{#each datasources}}
122
+ # {{displayName}}
123
+ aifabrix datasource test-e2e {{datasourceKey}} --app {{../appName}}
124
+
125
+ {{/each}}
126
+ ```
127
+ {{/if}}
128
+
129
+ Options:
130
+ -a, --app {{appName}} App key (default: resolve from cwd if inside integration/{{appName}}/)
131
+ -e, --env <env> Environment: dev, tst, or pro
132
+ -v, --verbose Show detailed step output and poll progress
133
+ --debug Include debug output and write log to integration/{{appName}}/logs/
134
+ --test-crud Enable CRUD lifecycle test (body testCrud: true)
135
+ --record-id <id> Record ID for test (body recordId)
136
+ --no-cleanup Disable cleanup after test (body cleanup: false)
137
+ --primary-key-value <value|@path> Primary key value or path to JSON file (e.g. @pk.json) for body primaryKeyValue
138
+ --no-async Use sync mode (no polling); single POST, no asyncRun
139
+
87
140
  ## Deployment
88
141
 
89
142
  Deploy via miso-controller pipeline (same as regular apps). Auth and controller come from `aifabrix login` and `aifabrix auth config`:
@@ -94,6 +147,6 @@ aifabrix deploy {{appName}}
94
147
 
95
148
  ## Troubleshooting
96
149
 
97
- - **Validation errors**: Run `aifabrix validate {{appName}} --type external` to see schema and manifest errors.
150
+ - **Validation errors**: Run `aifabrix validate {{appName}}` to see schema and manifest errors.
98
151
  - **Deployment / auth**: Run `aifabrix auth config --set-controller <url> --set-environment <env>` and `aifabrix login` before `aifabrix deploy`.
99
152
  - **File not found**: Run commands from the project root (where `package.json` and `integration/` live).
@@ -10,6 +10,8 @@ const { execSync } = require('child_process');
10
10
  const path = require('path');
11
11
 
12
12
  const scriptDir = __dirname;
13
+ // Project root (repo containing integration/ and builder/) so deploy/test-integration resolve app paths correctly
14
+ const projectRoot = path.join(scriptDir, '..', '..');
13
15
  const appKey = '{{systemKey}}';
14
16
  const env = process.env.ENVIRONMENT || 'dev';
15
17
  // Controller URL: from config (aifabrix auth config) or set CONTROLLER env before running
@@ -57,12 +59,12 @@ run('aifabrix validate "' + path.join(scriptDir, '{{this}}') + '"');
57
59
  console.log('✅ Validation passed');
58
60
 
59
61
  console.log('🚀 Deploying ' + appKey + '...');
60
- run('aifabrix deploy ' + appKey);
62
+ run('aifabrix deploy ' + appKey, { cwd: projectRoot });
61
63
  console.log('✅ Deployment complete');
62
64
 
63
65
  if (process.env.RUN_TESTS !== 'false') {
64
66
  console.log('🧪 Running integration tests...');
65
- run('aifabrix test-integration ' + appKey);
67
+ run('aifabrix test-integration ' + appKey, { cwd: projectRoot });
66
68
  console.log('✅ Tests passed');
67
69
  }
68
70