@aifabrix/builder 2.41.0 → 2.42.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/.cursor/rules/docs-rules.mdc +30 -0
  2. package/README.md +2 -2
  3. package/integration/hubspot/README.md +11 -5
  4. package/integration/hubspot/application.json +54 -0
  5. package/integration/hubspot/create-hubspot.js +9 -136
  6. package/integration/hubspot/env.template +3 -4
  7. package/integration/hubspot/hubspot-datasource-company.json +343 -5
  8. package/integration/hubspot/hubspot-datasource-contact.json +413 -5
  9. package/integration/hubspot/hubspot-datasource-deal.json +341 -4
  10. package/integration/hubspot/hubspot-datasource-users.json +116 -0
  11. package/integration/hubspot/hubspot-deploy.json +1250 -108
  12. package/integration/hubspot/hubspot-system.json +15 -32
  13. package/integration/hubspot/test-dataplane-down-tests.js +17 -16
  14. package/integration/hubspot/test-dataplane-down.js +2 -2
  15. package/jest.config.manual.js +2 -1
  16. package/lib/api/external-test.api.js +111 -0
  17. package/lib/api/index.js +42 -19
  18. package/lib/api/pipeline.api.js +66 -120
  19. package/lib/api/types/pipeline.types.js +37 -0
  20. package/lib/api/wizard-platform.api.js +61 -0
  21. package/lib/api/wizard.api.js +36 -2
  22. package/lib/app/config.js +23 -11
  23. package/lib/app/index.js +5 -3
  24. package/lib/app/prompts.js +46 -31
  25. package/lib/app/readme.js +11 -4
  26. package/lib/app/run-env-compose.js +64 -1
  27. package/lib/app/run-helpers.js +1 -1
  28. package/lib/app/show-display.js +1 -1
  29. package/lib/cli/setup-app.js +45 -14
  30. package/lib/cli/setup-credential-deployment.js +31 -6
  31. package/lib/cli/setup-dev.js +27 -0
  32. package/lib/cli/setup-environment.js +12 -4
  33. package/lib/cli/setup-external-system.js +19 -4
  34. package/lib/cli/setup-infra.js +54 -14
  35. package/lib/cli/setup-utility.js +117 -21
  36. package/lib/commands/auth-config.js +22 -12
  37. package/lib/commands/credential-env.js +162 -0
  38. package/lib/commands/credential-list.js +17 -22
  39. package/lib/commands/credential-push.js +96 -0
  40. package/lib/commands/datasource.js +77 -6
  41. package/lib/commands/dev-init.js +39 -1
  42. package/lib/commands/repair-auth-config.js +99 -0
  43. package/lib/commands/repair-datasource-keys.js +208 -0
  44. package/lib/commands/repair-datasource.js +235 -0
  45. package/lib/commands/repair-env-template.js +348 -0
  46. package/lib/commands/repair-internal.js +85 -0
  47. package/lib/commands/repair-rbac.js +158 -0
  48. package/lib/commands/repair.js +518 -0
  49. package/lib/commands/secrets-set.js +6 -0
  50. package/lib/commands/test-e2e-external.js +165 -0
  51. package/lib/commands/up-dataplane.js +90 -6
  52. package/lib/commands/upload.js +71 -40
  53. package/lib/commands/wizard-core-helpers.js +230 -5
  54. package/lib/commands/wizard-core.js +68 -29
  55. package/lib/commands/wizard-dataplane.js +1 -1
  56. package/lib/commands/wizard-entity-selection.js +43 -0
  57. package/lib/commands/wizard-headless.js +49 -5
  58. package/lib/commands/wizard-helpers.js +7 -3
  59. package/lib/commands/wizard.js +93 -64
  60. package/lib/core/config.js +7 -1
  61. package/lib/core/secrets.js +33 -12
  62. package/lib/datasource/deploy.js +12 -3
  63. package/lib/datasource/test-e2e.js +219 -0
  64. package/lib/datasource/test-integration.js +154 -0
  65. package/lib/deployment/deployer.js +7 -5
  66. package/lib/external-system/download-helpers.js +3 -1
  67. package/lib/external-system/download.js +182 -204
  68. package/lib/external-system/generator.js +204 -56
  69. package/lib/external-system/test-execution.js +2 -1
  70. package/lib/external-system/test-system-level.js +73 -0
  71. package/lib/external-system/test.js +51 -18
  72. package/lib/generator/external-controller-manifest.js +29 -2
  73. package/lib/generator/external-schema-utils.js +4 -2
  74. package/lib/generator/external.js +10 -3
  75. package/lib/generator/index.js +4 -1
  76. package/lib/generator/split-readme.js +1 -0
  77. package/lib/generator/split-variables.js +7 -1
  78. package/lib/generator/split.js +194 -54
  79. package/lib/generator/wizard-prompts-secondary.js +326 -0
  80. package/lib/generator/wizard-prompts.js +105 -106
  81. package/lib/generator/wizard-readme.js +91 -0
  82. package/lib/generator/wizard.js +180 -179
  83. package/lib/infrastructure/compose.js +11 -1
  84. package/lib/infrastructure/index.js +11 -3
  85. package/lib/infrastructure/services.js +22 -11
  86. package/lib/schema/application-schema.json +8 -5
  87. package/lib/schema/external-datasource.schema.json +49 -26
  88. package/lib/schema/external-system.schema.json +82 -6
  89. package/lib/schema/wizard-config.schema.json +23 -1
  90. package/lib/utils/api.js +38 -10
  91. package/lib/utils/auth-headers.js +8 -7
  92. package/lib/utils/compose-generator.js +1 -1
  93. package/lib/utils/compose-handlebars-helpers.js +11 -0
  94. package/lib/utils/config-format-preference.js +51 -0
  95. package/lib/utils/config-format.js +36 -0
  96. package/lib/utils/configuration-env-resolver.js +179 -0
  97. package/lib/utils/credential-display.js +83 -0
  98. package/lib/utils/credential-secrets-env.js +115 -25
  99. package/lib/utils/dataplane-pipeline-warning.js +28 -0
  100. package/lib/utils/deployment-validation-helpers.js +4 -4
  101. package/lib/utils/dev-ca-install.js +139 -0
  102. package/lib/utils/env-copy.js +23 -3
  103. package/lib/utils/error-formatters/http-status-errors.js +0 -1
  104. package/lib/utils/error-formatters/permission-errors.js +0 -1
  105. package/lib/utils/error-formatters/validation-errors.js +0 -1
  106. package/lib/utils/external-readme.js +89 -30
  107. package/lib/utils/external-system-display.js +59 -1
  108. package/lib/utils/external-system-test-helpers.js +21 -8
  109. package/lib/utils/external-system-validators.js +3 -0
  110. package/lib/utils/file-upload.js +20 -50
  111. package/lib/utils/help-builder.js +1 -0
  112. package/lib/utils/infra-status.js +50 -44
  113. package/lib/utils/local-secrets.js +5 -5
  114. package/lib/utils/paths.js +85 -4
  115. package/lib/utils/secrets-canonical.js +93 -0
  116. package/lib/utils/secrets-generator.js +20 -0
  117. package/lib/utils/secrets-helpers.js +75 -89
  118. package/lib/utils/test-log-writer.js +56 -0
  119. package/lib/utils/token-manager.js +24 -32
  120. package/lib/validation/env-template-auth.js +157 -0
  121. package/lib/validation/env-template-kv.js +41 -0
  122. package/lib/validation/external-manifest-validator.js +25 -0
  123. package/lib/validation/external-system-auth-rules.js +86 -0
  124. package/lib/validation/validate-batch.js +149 -0
  125. package/lib/validation/validate-datasource-keys-api.js +33 -0
  126. package/lib/validation/validate-display.js +94 -16
  127. package/lib/validation/validate.js +25 -12
  128. package/lib/validation/validator.js +7 -9
  129. package/lib/validation/wizard-datasource-validation.js +50 -0
  130. package/package.json +7 -2
  131. package/templates/applications/dataplane/application.yaml +1 -1
  132. package/templates/applications/dataplane/env.template +5 -5
  133. package/templates/applications/dataplane/rbac.yaml +2 -2
  134. package/templates/applications/miso-controller/env.template +1 -1
  135. package/templates/external-system/README.md.hbs +75 -22
  136. package/templates/external-system/deploy.js.hbs +4 -2
  137. package/templates/external-system/external-datasource.yaml.hbs +217 -0
  138. package/templates/external-system/external-system.json.hbs +1 -18
  139. package/templates/infra/compose.yaml.hbs +6 -0
  140. package/templates/python/docker-compose.hbs +4 -4
  141. package/templates/typescript/docker-compose.hbs +4 -4
  142. package/integration/hubspot/application.yaml +0 -37
@@ -0,0 +1,139 @@
1
+ /**
2
+ * Dev CA Install – SSL untrusted detection, fetch CA from Builder Server, install into OS trust store.
3
+ * Used by `aifabrix dev init` when the server certificate is self-signed. Only /install-ca uses
4
+ * rejectUnauthorized: false; all other requests use default TLS verification.
5
+ *
6
+ * @fileoverview CA install utilities for development Builder Server
7
+ * @author AI Fabrix Team
8
+ * @version 2.0.0
9
+ */
10
+
11
+ const fs = require('fs').promises;
12
+ const path = require('path');
13
+ const https = require('https');
14
+ const os = require('os');
15
+ const { execFileSync } = require('child_process');
16
+ const readline = require('readline');
17
+ const chalk = require('chalk');
18
+
19
+ const SSL_UNTRUSTED_CODES = [
20
+ 'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
21
+ 'DEPTH_ZERO_SELF_SIGNED_CERT',
22
+ 'CERT_UNTRUSTED',
23
+ 'SELF_SIGNED_CERT_IN_CHAIN',
24
+ 'UNABLE_TO_GET_ISSUER_CERT',
25
+ 'UNABLE_TO_GET_ISSUER_CERT_LOCALLY'
26
+ ];
27
+
28
+ /**
29
+ * Returns true if the error indicates an untrusted/self-signed server certificate.
30
+ * @param {Error} err - Thrown error (e.g. from devApi.getHealth)
31
+ * @returns {boolean}
32
+ */
33
+ function isSslUntrustedError(err) {
34
+ const code = err?.code || err?.cause?.code;
35
+ const msg = (err?.message || '').toUpperCase();
36
+ return SSL_UNTRUSTED_CODES.some(c => code === c || msg.includes(c));
37
+ }
38
+
39
+ /**
40
+ * Fetch CA PEM from Builder Server via GET {baseUrl}/install-ca.
41
+ * Uses rejectUnauthorized: false only for this endpoint (dev setup).
42
+ * @param {string} baseUrl - Builder Server base URL (no trailing slash)
43
+ * @returns {Promise<Buffer>} CA certificate PEM
44
+ */
45
+ function fetchInstallCa(baseUrl) {
46
+ return new Promise((resolve, reject) => {
47
+ const url = `${baseUrl.replace(/\/+$/, '')}/install-ca`;
48
+ const urlObj = new URL(url);
49
+ if (urlObj.protocol !== 'https:') {
50
+ reject(new Error('install-ca requires https URL'));
51
+ return;
52
+ }
53
+ const agent = new https.Agent({ rejectUnauthorized: false });
54
+ const req = https.get(
55
+ url,
56
+ { agent },
57
+ (res) => {
58
+ if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
59
+ req.destroy();
60
+ fetchInstallCa(res.headers.location).then(resolve).catch(reject);
61
+ return;
62
+ }
63
+ const chunks = [];
64
+ res.on('data', c => chunks.push(c));
65
+ res.on('end', () => {
66
+ const body = Buffer.concat(chunks).toString('utf8');
67
+ if (!body || !body.includes('-----BEGIN CERTIFICATE-----')) {
68
+ reject(new Error('Invalid CA response: expected PEM certificate'));
69
+ return;
70
+ }
71
+ resolve(Buffer.from(body, 'utf8'));
72
+ });
73
+ }
74
+ );
75
+ req.on('error', reject);
76
+ });
77
+ }
78
+
79
+ /**
80
+ * Install CA PEM into OS trust store (platform-specific).
81
+ * @param {Buffer|string} caPem - CA certificate PEM
82
+ * @param {string} baseUrl - Builder Server base URL (for help link)
83
+ * @returns {Promise<void>}
84
+ */
85
+ async function installCaPlatform(caPem, baseUrl) {
86
+ const pem = Buffer.isBuffer(caPem) ? caPem.toString('utf8') : String(caPem);
87
+ const tmpDir = os.tmpdir();
88
+ const tmpPath = path.join(tmpDir, 'aifabrix-root-ca.crt');
89
+ await fs.writeFile(tmpPath, pem, { mode: 0o644 });
90
+
91
+ try {
92
+ if (process.platform === 'win32') {
93
+ execFileSync('certutil', ['-addstore', '-user', 'ROOT', tmpPath], { stdio: 'inherit' });
94
+ } else if (process.platform === 'darwin') {
95
+ const keychain = path.join(os.homedir(), 'Library', 'Keychains', 'login.keychain-db');
96
+ execFileSync('security', ['add-trusted-cert', '-d', '-r', 'trustRoot', '-k', keychain, tmpPath], { stdio: 'inherit' });
97
+ } else if (process.platform === 'linux') {
98
+ const certPath = '/usr/local/share/ca-certificates/aifabrix-root-ca.crt';
99
+ try {
100
+ await fs.writeFile(certPath, pem, { mode: 0o644 });
101
+ execFileSync('update-ca-certificates', [], { stdio: 'inherit' });
102
+ } catch (e) {
103
+ if (e.code === 'EACCES' || (e.status !== undefined && e.status !== null && e.status !== 0)) {
104
+ const helpUrl = `${baseUrl.replace(/\/+$/, '')}/install-ca-help`;
105
+ throw new Error(
106
+ `Linux CA install requires sudo. Save CA manually from ${helpUrl} to ${certPath} and run: sudo update-ca-certificates`
107
+ );
108
+ }
109
+ throw e;
110
+ }
111
+ } else {
112
+ throw new Error(`Unsupported platform: ${process.platform}`);
113
+ }
114
+ } finally {
115
+ await fs.unlink(tmpPath).catch(() => {});
116
+ }
117
+ }
118
+
119
+ /**
120
+ * Prompt user: "Download and install the development CA? (y/n)"
121
+ * @returns {Promise<boolean>}
122
+ */
123
+ function promptInstallCa() {
124
+ const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
125
+ return new Promise(resolve => {
126
+ rl.question(chalk.yellow('Server certificate not trusted. Download and install the development CA? (y/n) '), answer => {
127
+ rl.close();
128
+ const normalized = (answer || '').trim().toLowerCase();
129
+ resolve(normalized === 'y' || normalized === 'yes');
130
+ });
131
+ });
132
+ }
133
+
134
+ module.exports = {
135
+ isSslUntrustedError,
136
+ fetchInstallCa,
137
+ installCaPlatform,
138
+ promptInstallCa
139
+ };
@@ -49,6 +49,22 @@ function readDeveloperIdFromConfig(config) {
49
49
  return null;
50
50
  }
51
51
 
52
+ /**
53
+ * Substitute /mnt/data with local mount path for local .env and ensure mount dir exists on disk.
54
+ * Creates the mount folder on the local filesystem (next to the .env file) when it does not exist.
55
+ * @param {string} content - Env file content
56
+ * @param {string} outputPath - Resolved path of the .env file being written
57
+ * @returns {string} Content with /mnt/data replaced by path to mount directory
58
+ */
59
+ function substituteMntDataForLocal(content, outputPath) {
60
+ const outputDir = path.dirname(outputPath);
61
+ const localMountPath = path.resolve(outputDir, 'mount');
62
+ if (!fs.existsSync(localMountPath)) {
63
+ fs.mkdirSync(localMountPath, { recursive: true });
64
+ }
65
+ return content.replace(/\/mnt\/data/g, localMountPath);
66
+ }
67
+
52
68
  /**
53
69
  * Resolve output path for env file
54
70
  * @param {string} rawOutputPath - Raw output path from application config
@@ -100,7 +116,8 @@ async function writeEnvOutputForReload(outputPath, runEnvPath) {
100
116
  */
101
117
  async function writeEnvOutputForLocal(appName, outputPath) {
102
118
  const { generateEnvContent, parseEnvContentToMap, mergeEnvMapIntoContent } = require('../core/secrets');
103
- const localContent = await generateEnvContent(appName, null, 'local', false);
119
+ let localContent = await generateEnvContent(appName, null, 'local', false);
120
+ localContent = substituteMntDataForLocal(localContent, outputPath);
104
121
  let toWrite = localContent;
105
122
  if (fs.existsSync(outputPath)) {
106
123
  const existingContent = await fsp.readFile(outputPath, 'utf8');
@@ -229,7 +246,8 @@ async function patchEnvContentForLocal(envContent, variables) {
229
246
  */
230
247
  async function writeLocalEnvToOutputPath(outputPath, appName, secretsPath, envOutputPathLabel) {
231
248
  const { generateEnvContent, parseEnvContentToMap, mergeEnvMapIntoContent } = require('../core/secrets');
232
- const localEnvContent = await generateEnvContent(appName, secretsPath, 'local', false);
249
+ let localEnvContent = await generateEnvContent(appName, secretsPath, 'local', false);
250
+ localEnvContent = substituteMntDataForLocal(localEnvContent, outputPath);
233
251
  let toWrite = localEnvContent;
234
252
  if (fs.existsSync(outputPath)) {
235
253
  const existingContent = fs.readFileSync(outputPath, 'utf8');
@@ -250,7 +268,8 @@ async function writeLocalEnvToOutputPath(outputPath, appName, secretsPath, envOu
250
268
  */
251
269
  async function writePatchedEnvToOutputPath(envPath, outputPath, variables, envOutputPathLabel) {
252
270
  const envContent = fs.readFileSync(envPath, 'utf8');
253
- const patchedContent = await patchEnvContentForLocal(envContent, variables);
271
+ let patchedContent = await patchEnvContentForLocal(envContent, variables);
272
+ patchedContent = substituteMntDataForLocal(patchedContent, outputPath);
254
273
  fs.writeFileSync(outputPath, patchedContent, { mode: 0o600 });
255
274
  logger.log(chalk.green(`✓ Copied .env to: ${envOutputPathLabel}`));
256
275
  }
@@ -292,6 +311,7 @@ async function processEnvVariables(envPath, variablesPath, appName, secretsPath)
292
311
  module.exports = {
293
312
  processEnvVariables,
294
313
  resolveEnvOutputPath,
314
+ substituteMntDataForLocal,
295
315
  writeEnvOutputForReload,
296
316
  writeEnvOutputForLocal
297
317
  };
@@ -323,4 +323,3 @@ module.exports = {
323
323
  formatNotFoundError,
324
324
  formatGenericError
325
325
  };
326
-
@@ -134,4 +134,3 @@ module.exports = {
134
134
  extractMissingPermissions,
135
135
  extractRequiredPermissions
136
136
  };
137
-
@@ -130,4 +130,3 @@ function formatValidationError(errorData) {
130
130
  module.exports = {
131
131
  formatValidationError
132
132
  };
133
-
@@ -30,42 +30,92 @@ function formatDisplayName(key) {
30
30
  .join(' ');
31
31
  }
32
32
 
33
+ /**
34
+ * Derives suffix from datasource key for filename generation
35
+ * @param {string} key - Datasource key
36
+ * @param {string} systemKey - System key
37
+ * @param {string} entityType - Fallback entity type
38
+ * @returns {string} Suffix segment
39
+ */
40
+ function getDatasourceKeySuffix(key, systemKey, entityType) {
41
+ if (key.startsWith(`${systemKey}-deploy-`)) {
42
+ return key.slice(`${systemKey}-deploy-`.length);
43
+ }
44
+ if (systemKey && key.startsWith(`${systemKey}-`)) {
45
+ return key.slice(systemKey.length + 1);
46
+ }
47
+ if (key) {
48
+ return key;
49
+ }
50
+ return entityType;
51
+ }
52
+
53
+ /**
54
+ * Normalizes a single datasource entry for template use
55
+ * @param {Object} datasource - Datasource object
56
+ * @param {number} index - Index in array
57
+ * @param {string} systemKey - System key for filename generation
58
+ * @param {string} ext - File extension (e.g. '.json', '.yaml')
59
+ * @returns {{entityType: string, displayName: string, fileName: string, datasourceKey: string}} Normalized entry
60
+ */
61
+ function normalizeOneDatasource(datasource, index, systemKey, ext) {
62
+ const entityType = datasource.entityType ||
63
+ datasource.entityKey ||
64
+ datasource.key?.split('-').pop() ||
65
+ `entity${index + 1}`;
66
+ const displayName = datasource.displayName ||
67
+ datasource.name ||
68
+ `Datasource ${index + 1}`;
69
+ const key = datasource.key || '';
70
+ const suffix = getDatasourceKeySuffix(key, systemKey, entityType);
71
+ const datasourceKey = key || (systemKey ? `${systemKey}-${suffix}` : suffix);
72
+ const fileName = datasource.fileName || datasource.file ||
73
+ (systemKey ? `${systemKey}-datasource-${suffix}${ext}` : `${suffix}${ext}`);
74
+ return { entityType, displayName, fileName, datasourceKey };
75
+ }
76
+
33
77
  /**
34
78
  * Normalizes datasource entries for template use
35
79
  * @param {Array} datasources - Datasource objects
36
80
  * @param {string} systemKey - System key for filename generation
37
- * @returns {Array<{entityType: string, displayName: string, fileName: string}>} Normalized entries
81
+ * @param {string} [fileExt='.json'] - File extension for generated filenames (e.g. '.json', '.yaml')
82
+ * @returns {Array<{entityType: string, displayName: string, fileName: string, datasourceKey: string}>} Normalized entries
38
83
  */
39
- function normalizeDatasources(datasources, systemKey) {
84
+ function normalizeDatasources(datasources, systemKey, fileExt = '.json') {
40
85
  if (!Array.isArray(datasources)) {
41
86
  return [];
42
87
  }
43
- return datasources.map((datasource, index) => {
44
- const entityType = datasource.entityType ||
45
- datasource.entityKey ||
46
- datasource.key?.split('-').pop() ||
47
- `entity${index + 1}`;
48
- const displayName = datasource.displayName ||
49
- datasource.name ||
50
- `Datasource ${index + 1}`;
51
- let fileName = datasource.fileName || datasource.file;
52
- if (!fileName) {
53
- const key = datasource.key || '';
54
- // Suffix matches split getExternalDatasourceFileName for consistent README and file names
55
- let suffix;
56
- if (key.startsWith(`${systemKey}-deploy-`)) {
57
- suffix = key.slice(`${systemKey}-deploy-`.length);
58
- } else if (systemKey && key.startsWith(`${systemKey}-`)) {
59
- suffix = key.slice(systemKey.length + 1);
60
- } else if (key) {
61
- suffix = key;
62
- } else {
63
- suffix = entityType;
64
- }
65
- fileName = systemKey ? `${systemKey}-datasource-${suffix}.yaml` : `${suffix}.yaml`;
66
- }
67
- return { entityType, displayName, fileName };
68
- });
88
+ const ext = fileExt && fileExt.startsWith('.') ? fileExt : `.${fileExt || 'json'}`;
89
+ return datasources.map((datasource, index) =>
90
+ normalizeOneDatasource(datasource, index, systemKey, ext)
91
+ );
92
+ }
93
+
94
+ /**
95
+ * Builds secret path entries for README "Secrets" section per auth type.
96
+ * Path is the key for `aifabrix secret set <key> <value>` (no kv:// prefix; key format systemKey/secretKey in camelCase).
97
+ * @param {string} systemKey - System key
98
+ * @param {string} [authType] - Authentication type (oauth2, aad, apikey, basic, queryParam, hmac, bearer, token, none)
99
+ * @returns {Array<{path: string, description: string}>} secretPaths for template (path = key for secret set, no kv://)
100
+ */
101
+ function buildSecretPaths(systemKey, authType) {
102
+ if (!systemKey || typeof systemKey !== 'string') return [];
103
+ const t = (authType && typeof authType === 'string') ? authType.toLowerCase() : 'apikey';
104
+ const map = {
105
+ oauth2: [{ path: `${systemKey}/clientId`, description: 'Client ID' }, { path: `${systemKey}/clientSecret`, description: 'Client Secret' }],
106
+ oauth: [{ path: `${systemKey}/clientId`, description: 'Client ID' }, { path: `${systemKey}/clientSecret`, description: 'Client Secret' }],
107
+ aad: [{ path: `${systemKey}/clientId`, description: 'Client ID' }, { path: `${systemKey}/clientSecret`, description: 'Client Secret' }],
108
+ apikey: [{ path: `${systemKey}/apiKey`, description: 'API Key' }],
109
+ apiKey: [{ path: `${systemKey}/apiKey`, description: 'API Key' }],
110
+ basic: [{ path: `${systemKey}/username`, description: 'Username' }, { path: `${systemKey}/password`, description: 'Password' }],
111
+ queryparam: [{ path: `${systemKey}/paramValue`, description: 'Query parameter value' }],
112
+ hmac: [{ path: `${systemKey}/signingSecret`, description: 'Signing secret' }],
113
+ bearer: [{ path: `${systemKey}/bearerToken`, description: 'Bearer token' }],
114
+ token: [{ path: `${systemKey}/bearerToken`, description: 'Bearer token' }],
115
+ oidc: [],
116
+ none: []
117
+ };
118
+ return map[t] || map.apikey;
69
119
  }
70
120
 
71
121
  /**
@@ -78,6 +128,9 @@ function normalizeDatasources(datasources, systemKey) {
78
128
  * @param {string} [params.displayName] - Display name
79
129
  * @param {string} [params.description] - Description
80
130
  * @param {Array} [params.datasources] - Datasource objects
131
+ * @param {string} [params.fileExt] - File extension for config files (e.g. '.json', '.yaml'); default '.json'
132
+ * @param {string} [params.authType] - Authentication type for Secrets section (oauth2, aad, apikey, basic, etc.)
133
+ * @param {Object} [params.authentication] - Full authentication object (authType used if authType not set)
81
134
  * @returns {Object} Template context
82
135
  */
83
136
  function buildExternalReadmeContext(params = {}) {
@@ -86,7 +139,10 @@ function buildExternalReadmeContext(params = {}) {
86
139
  const displayName = params.displayName || formatDisplayName(systemKey);
87
140
  const description = params.description || `External system integration for ${systemKey}`;
88
141
  const systemType = params.systemType || 'openapi';
89
- const datasources = normalizeDatasources(params.datasources, systemKey);
142
+ const fileExt = params.fileExt !== undefined ? params.fileExt : '.json';
143
+ const datasources = normalizeDatasources(params.datasources, systemKey, fileExt);
144
+ const authType = params.authType || params.authentication?.type || params.authentication?.method || params.authentication?.authType;
145
+ const secretPaths = buildSecretPaths(systemKey, authType);
90
146
 
91
147
  return {
92
148
  appName,
@@ -94,8 +150,11 @@ function buildExternalReadmeContext(params = {}) {
94
150
  displayName,
95
151
  description,
96
152
  systemType,
153
+ fileExt: fileExt && fileExt.startsWith('.') ? fileExt : `.${fileExt || 'json'}`,
97
154
  datasourceCount: datasources.length,
98
- datasources
155
+ hasDatasources: datasources.length > 0,
156
+ datasources,
157
+ secretPaths
99
158
  };
100
159
  }
101
160
 
@@ -241,8 +241,66 @@ function displayIntegrationTestResults(results, verbose = false) {
241
241
  }
242
242
  }
243
243
 
244
+ /**
245
+ * Displays E2E test results (steps: config, credential, sync, data, cip).
246
+ * Supports sync response (data.steps only), final poll (data.steps + data.success), and running poll
247
+ * (data.completedActions, no data.steps yet). When status is present (async flow), shows it.
248
+ *
249
+ * @param {Object} data - E2E response or poll data
250
+ * @param {string} [data.status] - Optional status: 'running' | 'completed' | 'failed' (async flow)
251
+ * @param {Object[]} [data.steps] - Per-step results (final state)
252
+ * @param {Object[]} [data.completedActions] - Steps completed so far (running state when steps absent)
253
+ * @param {boolean} [data.success] - Overall success (final state)
254
+ * @param {string} [data.error] - Error message when failed
255
+ * @param {boolean} [verbose] - Show detailed output
256
+ */
257
+ /* eslint-disable max-statements,complexity -- Step iteration and status display */
258
+ function displayE2EResults(data, verbose = false) {
259
+ logger.log(chalk.blue('\n📊 E2E Test Results\n'));
260
+ if (data.status) {
261
+ const statusLabel = data.status === 'running'
262
+ ? chalk.yellow('running')
263
+ : data.status === 'completed'
264
+ ? chalk.green('completed')
265
+ : data.status === 'failed'
266
+ ? chalk.red('failed')
267
+ : data.status;
268
+ logger.log(`Status: ${statusLabel}`);
269
+ }
270
+ const steps = data.steps || data.completedActions || [];
271
+ if (steps.length === 0) {
272
+ if (data.success === false) {
273
+ logger.log(chalk.red('✗ E2E test failed'));
274
+ if (data.error) logger.log(chalk.red(` Error: ${data.error}`));
275
+ } else if (data.status === 'running') {
276
+ logger.log(chalk.gray(' No steps completed yet'));
277
+ } else {
278
+ logger.log(chalk.yellow('No step results returned'));
279
+ }
280
+ return;
281
+ }
282
+ const isRunning = data.status === 'running' && !data.steps;
283
+ if (isRunning && verbose) {
284
+ logger.log(chalk.gray(` (${steps.length} step(s) completed so far)`));
285
+ }
286
+ for (const step of steps) {
287
+ const name = step.name || step.step || 'unknown';
288
+ const ok = step.success !== false && !step.error;
289
+ logger.log(` ${ok ? chalk.green('✓') : chalk.red('✗')} ${name}`);
290
+ if (!ok && (step.error || step.message)) logger.log(chalk.red(` ${step.error || step.message}`));
291
+ if (verbose && step.message && ok) logger.log(chalk.gray(` ${step.message}`));
292
+ }
293
+ if (isRunning) {
294
+ return;
295
+ }
296
+ const allPassed = steps.every(s => s.success !== false && !s.error);
297
+ logger.log(allPassed ? chalk.green('\n✅ E2E test passed!') : chalk.red('\n❌ E2E test failed'));
298
+ }
299
+
244
300
  module.exports = {
245
301
  displayTestResults,
246
- displayIntegrationTestResults
302
+ displayIntegrationTestResults,
303
+ displayE2EResults,
304
+ displayDatasourceIntegrationResult
247
305
  };
248
306
 
@@ -12,7 +12,8 @@
12
12
  const fs = require('fs').promises;
13
13
  const path = require('path');
14
14
  const { testDatasourceViaPipeline } = require('../api/pipeline.api');
15
- const { requireBearerForDataplanePipeline } = require('./token-manager');
15
+
16
+ /** Pipeline test endpoints accept client credentials; do not enforce Bearer-only */
16
17
 
17
18
  /**
18
19
  * Retry API call with exponential backoff
@@ -40,26 +41,31 @@ async function retryApiCall(fn, maxRetries = 3, backoffMs = 1000) {
40
41
  }
41
42
 
42
43
  /**
43
- * Calls pipeline test endpoint using centralized API client
44
+ * Calls pipeline test endpoint using centralized API client.
45
+ * Pipeline test accepts Bearer, API_KEY, or client credentials (x-client-id/x-client-secret) for CI/CD.
44
46
  * @async
45
47
  * @param {Object} params - Function parameters
46
48
  * @param {string} params.systemKey - System key
47
49
  * @param {string} params.datasourceKey - Datasource key
48
50
  * @param {Object} params.payloadTemplate - Test payload template
49
51
  * @param {string} params.dataplaneUrl - Dataplane URL
50
- * @param {Object} params.authConfig - Authentication configuration
52
+ * @param {Object} params.authConfig - Authentication configuration (token or client credentials)
51
53
  * @param {number} [params.timeout] - Request timeout in milliseconds (default: 30000)
54
+ * @param {boolean} [params.includeDebug] - Include debug output in response
52
55
  * @returns {Promise<Object>} Test response
53
56
  */
54
- async function callPipelineTestEndpoint({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout = 30000 }) {
55
- requireBearerForDataplanePipeline(authConfig);
57
+ async function callPipelineTestEndpoint({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout = 30000, includeDebug = false }) {
58
+ const testData = { payloadTemplate };
59
+ if (includeDebug) {
60
+ testData.includeDebug = true;
61
+ }
56
62
  const response = await retryApiCall(async() => {
57
63
  return await testDatasourceViaPipeline({
58
64
  dataplaneUrl,
59
65
  systemKey,
60
66
  datasourceKey,
61
67
  authConfig,
62
- testData: { payloadTemplate },
68
+ testData,
63
69
  options: { timeout }
64
70
  });
65
71
  });
@@ -67,6 +73,11 @@ async function callPipelineTestEndpoint({ systemKey, datasourceKey, payloadTempl
67
73
  if (!response.success || !response.data) {
68
74
  throw new Error(`Test endpoint failed: ${response.error || response.formattedError || 'Unknown error'}`);
69
75
  }
76
+ // When 200 with success: false in body, pass through; caller interprets via data.success
77
+ if (response.data?.success === false) {
78
+ const errMsg = response.data?.error || response.data?.formattedError || 'Test failed';
79
+ throw new Error(`Test endpoint failed: ${errMsg}`);
80
+ }
70
81
 
71
82
  return response.data.data || response.data;
72
83
  }
@@ -114,16 +125,18 @@ function determinePayloadTemplate(datasource, datasourceKey, customPayload) {
114
125
  * @param {string} params.dataplaneUrl - Dataplane URL
115
126
  * @param {Object} params.authConfig - Authentication configuration
116
127
  * @param {number} params.timeout - Request timeout
128
+ * @param {boolean} [params.includeDebug] - Include debug in response
117
129
  * @returns {Promise<Object>} Test result
118
130
  */
119
- async function testSingleDatasource({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout }) {
131
+ async function testSingleDatasource({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout, includeDebug = false }) {
120
132
  const testResponse = await callPipelineTestEndpoint({
121
133
  systemKey,
122
134
  datasourceKey,
123
135
  payloadTemplate,
124
136
  dataplaneUrl,
125
137
  authConfig,
126
- timeout
138
+ timeout,
139
+ includeDebug
127
140
  });
128
141
 
129
142
  return {
@@ -9,6 +9,7 @@
9
9
  */
10
10
 
11
11
  const Ajv = require('ajv');
12
+ const addFormats = require('ajv-formats');
12
13
 
13
14
  /**
14
15
  * Validates field mapping expression syntax (pipe-based DSL)
@@ -263,6 +264,7 @@ function validateMetadataSchema(datasource, testPayload) {
263
264
 
264
265
  try {
265
266
  const ajv = new Ajv({ allErrors: true, strict: false });
267
+ addFormats(ajv);
266
268
  const validate = ajv.compile(datasource.metadataSchema);
267
269
  const valid = validate(payloadTemplate);
268
270
 
@@ -319,6 +321,7 @@ function validateAgainstSchema(data, schema) {
319
321
  allowUnionTypes: true,
320
322
  validateSchema: false
321
323
  });
324
+ addFormats(ajv);
322
325
  // Remove $schema for draft-2020-12 to avoid AJV issues
323
326
  const schemaCopy = { ...schema };
324
327
  if (schemaCopy.$schema && schemaCopy.$schema.includes('2020-12')) {
@@ -1,29 +1,14 @@
1
1
  /**
2
2
  * @fileoverview File upload utilities for multipart/form-data requests
3
+ * All API calls go via ApiClient (lib/api/index.js); no duplicate auth logic.
3
4
  * @author AI Fabrix Team
4
5
  * @version 2.0.0
5
6
  */
6
7
 
7
8
  const fs = require('fs').promises;
8
9
  const path = require('path');
9
- const { makeApiCall, authenticatedApiCall } = require('./api');
10
+ const { ApiClient } = require('../api');
10
11
 
11
- /**
12
- * Upload a file using multipart/form-data
13
- * @async
14
- * @function uploadFile
15
- * @param {string} url - API endpoint URL
16
- * @param {string} filePath - Path to file to upload
17
- * @param {string} fieldName - Form field name for the file (default: 'file')
18
- * @param {Object} [authConfig] - Authentication configuration
19
- * @param {string} [authConfig.type] - Auth type ('bearer' | 'client-credentials')
20
- * @param {string} [authConfig.token] - Bearer token
21
- * @param {string} [authConfig.clientId] - Client ID
22
- * @param {string} [authConfig.clientSecret] - Client secret
23
- * @param {Object} [additionalFields] - Additional form fields to include
24
- * @returns {Promise<Object>} API response
25
- * @throws {Error} If file upload fails
26
- */
27
12
  /**
28
13
  * Validates file exists
29
14
  * @async
@@ -63,47 +48,32 @@ async function buildFormData(filePath, fieldName, additionalFields) {
63
48
  }
64
49
 
65
50
  /**
66
- * Builds authentication headers
67
- * @function buildAuthHeaders
68
- * @param {Object} authConfig - Authentication configuration
69
- * @returns {Object} Headers object
51
+ * Upload a file using multipart/form-data via ApiClient (single place for auth and API calls).
52
+ * @async
53
+ * @function uploadFile
54
+ * @param {string} url - Full API endpoint URL (e.g. https://dataplane.example.com/api/v1/wizard/parse-openapi)
55
+ * @param {string} filePath - Path to file to upload
56
+ * @param {string} fieldName - Form field name for the file (default: 'file')
57
+ * @param {Object} [authConfig] - Authentication configuration (token-only for app endpoints)
58
+ * @param {string} [authConfig.type] - Auth type ('bearer' | 'client-token')
59
+ * @param {string} [authConfig.token] - Token (Bearer user token or x-client-token application token)
60
+ * @param {Object} [additionalFields] - Additional form fields to include
61
+ * @returns {Promise<Object>} API response
62
+ * @throws {Error} If file upload fails
70
63
  */
71
- function buildAuthHeaders(authConfig) {
72
- const headers = {};
73
- if (authConfig.type === 'bearer' && authConfig.token) {
74
- headers['Authorization'] = `Bearer ${authConfig.token}`;
75
- } else if (authConfig.type === 'client-credentials') {
76
- if (authConfig.clientId) {
77
- headers['x-client-id'] = authConfig.clientId;
78
- }
79
- if (authConfig.clientSecret) {
80
- headers['x-client-secret'] = authConfig.clientSecret;
81
- }
82
- }
83
- return headers;
84
- }
85
-
86
64
  async function uploadFile(url, filePath, fieldName = 'file', authConfig = {}, additionalFields = {}) {
87
65
  await validateFileExists(filePath);
88
66
 
89
- const formData = await buildFormData(filePath, fieldName, additionalFields);
90
- const headers = buildAuthHeaders(authConfig);
91
-
92
- const options = {
93
- method: 'POST',
94
- headers,
95
- body: formData
96
- };
67
+ const parsed = new URL(url);
68
+ const baseUrl = parsed.origin;
69
+ const endpointPath = parsed.pathname + parsed.search;
97
70
 
98
- // Use authenticatedApiCall if bearer token, otherwise makeApiCall
99
- if (authConfig.type === 'bearer' && authConfig.token) {
100
- return await authenticatedApiCall(url, options, authConfig.token);
101
- }
71
+ const formData = await buildFormData(filePath, fieldName, additionalFields);
72
+ const client = new ApiClient(baseUrl, authConfig);
102
73
 
103
- return await makeApiCall(url, options);
74
+ return await client.postFormData(endpointPath, formData);
104
75
  }
105
76
 
106
77
  module.exports = {
107
78
  uploadFile
108
79
  };
109
-
@@ -97,6 +97,7 @@ const CATEGORIES = [
97
97
  { name: 'download', term: 'download <system-key>' },
98
98
  { name: 'upload', term: 'upload <system-key>' },
99
99
  { name: 'delete', term: 'delete <system-key>' },
100
+ { name: 'repair', term: 'repair <app>' },
100
101
  { name: 'test', term: 'test <app>' },
101
102
  { name: 'test-integration', term: 'test-integration <app>' }
102
103
  ]