@aifabrix/builder 2.41.0 → 2.42.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursor/rules/docs-rules.mdc +30 -0
- package/README.md +2 -2
- package/integration/hubspot/README.md +11 -5
- package/integration/hubspot/application.json +54 -0
- package/integration/hubspot/create-hubspot.js +9 -136
- package/integration/hubspot/env.template +3 -4
- package/integration/hubspot/hubspot-datasource-company.json +343 -5
- package/integration/hubspot/hubspot-datasource-contact.json +413 -5
- package/integration/hubspot/hubspot-datasource-deal.json +341 -4
- package/integration/hubspot/hubspot-datasource-users.json +116 -0
- package/integration/hubspot/hubspot-deploy.json +1250 -108
- package/integration/hubspot/hubspot-system.json +15 -32
- package/integration/hubspot/test-dataplane-down-tests.js +17 -16
- package/integration/hubspot/test-dataplane-down.js +2 -2
- package/jest.config.manual.js +2 -1
- package/lib/api/external-test.api.js +111 -0
- package/lib/api/index.js +42 -19
- package/lib/api/pipeline.api.js +66 -120
- package/lib/api/types/pipeline.types.js +37 -0
- package/lib/api/wizard-platform.api.js +61 -0
- package/lib/api/wizard.api.js +36 -2
- package/lib/app/config.js +23 -11
- package/lib/app/index.js +5 -3
- package/lib/app/prompts.js +46 -31
- package/lib/app/readme.js +11 -4
- package/lib/app/run-env-compose.js +64 -1
- package/lib/app/run-helpers.js +1 -1
- package/lib/app/show-display.js +1 -1
- package/lib/cli/setup-app.js +45 -14
- package/lib/cli/setup-credential-deployment.js +31 -6
- package/lib/cli/setup-dev.js +27 -0
- package/lib/cli/setup-environment.js +12 -4
- package/lib/cli/setup-external-system.js +19 -4
- package/lib/cli/setup-infra.js +54 -14
- package/lib/cli/setup-utility.js +117 -21
- package/lib/commands/auth-config.js +22 -12
- package/lib/commands/credential-env.js +162 -0
- package/lib/commands/credential-list.js +17 -22
- package/lib/commands/credential-push.js +96 -0
- package/lib/commands/datasource.js +77 -6
- package/lib/commands/dev-init.js +39 -1
- package/lib/commands/repair-auth-config.js +99 -0
- package/lib/commands/repair-datasource-keys.js +208 -0
- package/lib/commands/repair-datasource.js +235 -0
- package/lib/commands/repair-env-template.js +348 -0
- package/lib/commands/repair-internal.js +85 -0
- package/lib/commands/repair-rbac.js +158 -0
- package/lib/commands/repair.js +518 -0
- package/lib/commands/secrets-set.js +6 -0
- package/lib/commands/test-e2e-external.js +165 -0
- package/lib/commands/up-dataplane.js +90 -6
- package/lib/commands/upload.js +71 -40
- package/lib/commands/wizard-core-helpers.js +230 -5
- package/lib/commands/wizard-core.js +68 -29
- package/lib/commands/wizard-dataplane.js +1 -1
- package/lib/commands/wizard-entity-selection.js +43 -0
- package/lib/commands/wizard-headless.js +49 -5
- package/lib/commands/wizard-helpers.js +7 -3
- package/lib/commands/wizard.js +93 -64
- package/lib/core/config.js +7 -1
- package/lib/core/secrets.js +33 -12
- package/lib/datasource/deploy.js +12 -3
- package/lib/datasource/test-e2e.js +219 -0
- package/lib/datasource/test-integration.js +154 -0
- package/lib/deployment/deployer.js +7 -5
- package/lib/external-system/download-helpers.js +3 -1
- package/lib/external-system/download.js +182 -204
- package/lib/external-system/generator.js +204 -56
- package/lib/external-system/test-execution.js +2 -1
- package/lib/external-system/test-system-level.js +73 -0
- package/lib/external-system/test.js +51 -18
- package/lib/generator/external-controller-manifest.js +29 -2
- package/lib/generator/external-schema-utils.js +4 -2
- package/lib/generator/external.js +10 -3
- package/lib/generator/index.js +4 -1
- package/lib/generator/split-readme.js +1 -0
- package/lib/generator/split-variables.js +7 -1
- package/lib/generator/split.js +194 -54
- package/lib/generator/wizard-prompts-secondary.js +326 -0
- package/lib/generator/wizard-prompts.js +105 -106
- package/lib/generator/wizard-readme.js +91 -0
- package/lib/generator/wizard.js +180 -179
- package/lib/infrastructure/compose.js +11 -1
- package/lib/infrastructure/index.js +11 -3
- package/lib/infrastructure/services.js +22 -11
- package/lib/schema/application-schema.json +8 -5
- package/lib/schema/external-datasource.schema.json +49 -26
- package/lib/schema/external-system.schema.json +82 -6
- package/lib/schema/wizard-config.schema.json +23 -1
- package/lib/utils/api.js +38 -10
- package/lib/utils/auth-headers.js +8 -7
- package/lib/utils/compose-generator.js +1 -1
- package/lib/utils/compose-handlebars-helpers.js +11 -0
- package/lib/utils/config-format-preference.js +51 -0
- package/lib/utils/config-format.js +36 -0
- package/lib/utils/configuration-env-resolver.js +179 -0
- package/lib/utils/credential-display.js +83 -0
- package/lib/utils/credential-secrets-env.js +115 -25
- package/lib/utils/dataplane-pipeline-warning.js +28 -0
- package/lib/utils/deployment-validation-helpers.js +4 -4
- package/lib/utils/dev-ca-install.js +139 -0
- package/lib/utils/env-copy.js +23 -3
- package/lib/utils/error-formatters/http-status-errors.js +0 -1
- package/lib/utils/error-formatters/permission-errors.js +0 -1
- package/lib/utils/error-formatters/validation-errors.js +0 -1
- package/lib/utils/external-readme.js +89 -30
- package/lib/utils/external-system-display.js +59 -1
- package/lib/utils/external-system-test-helpers.js +21 -8
- package/lib/utils/external-system-validators.js +3 -0
- package/lib/utils/file-upload.js +20 -50
- package/lib/utils/help-builder.js +1 -0
- package/lib/utils/infra-status.js +50 -44
- package/lib/utils/local-secrets.js +5 -5
- package/lib/utils/paths.js +85 -4
- package/lib/utils/secrets-canonical.js +93 -0
- package/lib/utils/secrets-generator.js +20 -0
- package/lib/utils/secrets-helpers.js +75 -89
- package/lib/utils/test-log-writer.js +56 -0
- package/lib/utils/token-manager.js +24 -32
- package/lib/validation/env-template-auth.js +157 -0
- package/lib/validation/env-template-kv.js +41 -0
- package/lib/validation/external-manifest-validator.js +25 -0
- package/lib/validation/external-system-auth-rules.js +86 -0
- package/lib/validation/validate-batch.js +149 -0
- package/lib/validation/validate-datasource-keys-api.js +33 -0
- package/lib/validation/validate-display.js +94 -16
- package/lib/validation/validate.js +25 -12
- package/lib/validation/validator.js +7 -9
- package/lib/validation/wizard-datasource-validation.js +50 -0
- package/package.json +7 -2
- package/templates/applications/dataplane/application.yaml +1 -1
- package/templates/applications/dataplane/env.template +5 -5
- package/templates/applications/dataplane/rbac.yaml +2 -2
- package/templates/applications/miso-controller/env.template +1 -1
- package/templates/external-system/README.md.hbs +75 -22
- package/templates/external-system/deploy.js.hbs +4 -2
- package/templates/external-system/external-datasource.yaml.hbs +217 -0
- package/templates/external-system/external-system.json.hbs +1 -18
- package/templates/infra/compose.yaml.hbs +6 -0
- package/templates/python/docker-compose.hbs +4 -4
- package/templates/typescript/docker-compose.hbs +4 -4
- package/integration/hubspot/application.yaml +0 -37
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dev CA Install – SSL untrusted detection, fetch CA from Builder Server, install into OS trust store.
|
|
3
|
+
* Used by `aifabrix dev init` when the server certificate is self-signed. Only /install-ca uses
|
|
4
|
+
* rejectUnauthorized: false; all other requests use default TLS verification.
|
|
5
|
+
*
|
|
6
|
+
* @fileoverview CA install utilities for development Builder Server
|
|
7
|
+
* @author AI Fabrix Team
|
|
8
|
+
* @version 2.0.0
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const fs = require('fs').promises;
|
|
12
|
+
const path = require('path');
|
|
13
|
+
const https = require('https');
|
|
14
|
+
const os = require('os');
|
|
15
|
+
const { execFileSync } = require('child_process');
|
|
16
|
+
const readline = require('readline');
|
|
17
|
+
const chalk = require('chalk');
|
|
18
|
+
|
|
19
|
+
const SSL_UNTRUSTED_CODES = [
|
|
20
|
+
'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
|
|
21
|
+
'DEPTH_ZERO_SELF_SIGNED_CERT',
|
|
22
|
+
'CERT_UNTRUSTED',
|
|
23
|
+
'SELF_SIGNED_CERT_IN_CHAIN',
|
|
24
|
+
'UNABLE_TO_GET_ISSUER_CERT',
|
|
25
|
+
'UNABLE_TO_GET_ISSUER_CERT_LOCALLY'
|
|
26
|
+
];
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Returns true if the error indicates an untrusted/self-signed server certificate.
|
|
30
|
+
* @param {Error} err - Thrown error (e.g. from devApi.getHealth)
|
|
31
|
+
* @returns {boolean}
|
|
32
|
+
*/
|
|
33
|
+
function isSslUntrustedError(err) {
|
|
34
|
+
const code = err?.code || err?.cause?.code;
|
|
35
|
+
const msg = (err?.message || '').toUpperCase();
|
|
36
|
+
return SSL_UNTRUSTED_CODES.some(c => code === c || msg.includes(c));
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Fetch CA PEM from Builder Server via GET {baseUrl}/install-ca.
|
|
41
|
+
* Uses rejectUnauthorized: false only for this endpoint (dev setup).
|
|
42
|
+
* @param {string} baseUrl - Builder Server base URL (no trailing slash)
|
|
43
|
+
* @returns {Promise<Buffer>} CA certificate PEM
|
|
44
|
+
*/
|
|
45
|
+
function fetchInstallCa(baseUrl) {
|
|
46
|
+
return new Promise((resolve, reject) => {
|
|
47
|
+
const url = `${baseUrl.replace(/\/+$/, '')}/install-ca`;
|
|
48
|
+
const urlObj = new URL(url);
|
|
49
|
+
if (urlObj.protocol !== 'https:') {
|
|
50
|
+
reject(new Error('install-ca requires https URL'));
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
const agent = new https.Agent({ rejectUnauthorized: false });
|
|
54
|
+
const req = https.get(
|
|
55
|
+
url,
|
|
56
|
+
{ agent },
|
|
57
|
+
(res) => {
|
|
58
|
+
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
59
|
+
req.destroy();
|
|
60
|
+
fetchInstallCa(res.headers.location).then(resolve).catch(reject);
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
const chunks = [];
|
|
64
|
+
res.on('data', c => chunks.push(c));
|
|
65
|
+
res.on('end', () => {
|
|
66
|
+
const body = Buffer.concat(chunks).toString('utf8');
|
|
67
|
+
if (!body || !body.includes('-----BEGIN CERTIFICATE-----')) {
|
|
68
|
+
reject(new Error('Invalid CA response: expected PEM certificate'));
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
resolve(Buffer.from(body, 'utf8'));
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
);
|
|
75
|
+
req.on('error', reject);
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Install CA PEM into OS trust store (platform-specific).
|
|
81
|
+
* @param {Buffer|string} caPem - CA certificate PEM
|
|
82
|
+
* @param {string} baseUrl - Builder Server base URL (for help link)
|
|
83
|
+
* @returns {Promise<void>}
|
|
84
|
+
*/
|
|
85
|
+
async function installCaPlatform(caPem, baseUrl) {
|
|
86
|
+
const pem = Buffer.isBuffer(caPem) ? caPem.toString('utf8') : String(caPem);
|
|
87
|
+
const tmpDir = os.tmpdir();
|
|
88
|
+
const tmpPath = path.join(tmpDir, 'aifabrix-root-ca.crt');
|
|
89
|
+
await fs.writeFile(tmpPath, pem, { mode: 0o644 });
|
|
90
|
+
|
|
91
|
+
try {
|
|
92
|
+
if (process.platform === 'win32') {
|
|
93
|
+
execFileSync('certutil', ['-addstore', '-user', 'ROOT', tmpPath], { stdio: 'inherit' });
|
|
94
|
+
} else if (process.platform === 'darwin') {
|
|
95
|
+
const keychain = path.join(os.homedir(), 'Library', 'Keychains', 'login.keychain-db');
|
|
96
|
+
execFileSync('security', ['add-trusted-cert', '-d', '-r', 'trustRoot', '-k', keychain, tmpPath], { stdio: 'inherit' });
|
|
97
|
+
} else if (process.platform === 'linux') {
|
|
98
|
+
const certPath = '/usr/local/share/ca-certificates/aifabrix-root-ca.crt';
|
|
99
|
+
try {
|
|
100
|
+
await fs.writeFile(certPath, pem, { mode: 0o644 });
|
|
101
|
+
execFileSync('update-ca-certificates', [], { stdio: 'inherit' });
|
|
102
|
+
} catch (e) {
|
|
103
|
+
if (e.code === 'EACCES' || (e.status !== undefined && e.status !== null && e.status !== 0)) {
|
|
104
|
+
const helpUrl = `${baseUrl.replace(/\/+$/, '')}/install-ca-help`;
|
|
105
|
+
throw new Error(
|
|
106
|
+
`Linux CA install requires sudo. Save CA manually from ${helpUrl} to ${certPath} and run: sudo update-ca-certificates`
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
throw e;
|
|
110
|
+
}
|
|
111
|
+
} else {
|
|
112
|
+
throw new Error(`Unsupported platform: ${process.platform}`);
|
|
113
|
+
}
|
|
114
|
+
} finally {
|
|
115
|
+
await fs.unlink(tmpPath).catch(() => {});
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Prompt user: "Download and install the development CA? (y/n)"
|
|
121
|
+
* @returns {Promise<boolean>}
|
|
122
|
+
*/
|
|
123
|
+
function promptInstallCa() {
|
|
124
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
125
|
+
return new Promise(resolve => {
|
|
126
|
+
rl.question(chalk.yellow('Server certificate not trusted. Download and install the development CA? (y/n) '), answer => {
|
|
127
|
+
rl.close();
|
|
128
|
+
const normalized = (answer || '').trim().toLowerCase();
|
|
129
|
+
resolve(normalized === 'y' || normalized === 'yes');
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
module.exports = {
|
|
135
|
+
isSslUntrustedError,
|
|
136
|
+
fetchInstallCa,
|
|
137
|
+
installCaPlatform,
|
|
138
|
+
promptInstallCa
|
|
139
|
+
};
|
package/lib/utils/env-copy.js
CHANGED
|
@@ -49,6 +49,22 @@ function readDeveloperIdFromConfig(config) {
|
|
|
49
49
|
return null;
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
+
/**
|
|
53
|
+
* Substitute /mnt/data with local mount path for local .env and ensure mount dir exists on disk.
|
|
54
|
+
* Creates the mount folder on the local filesystem (next to the .env file) when it does not exist.
|
|
55
|
+
* @param {string} content - Env file content
|
|
56
|
+
* @param {string} outputPath - Resolved path of the .env file being written
|
|
57
|
+
* @returns {string} Content with /mnt/data replaced by path to mount directory
|
|
58
|
+
*/
|
|
59
|
+
function substituteMntDataForLocal(content, outputPath) {
|
|
60
|
+
const outputDir = path.dirname(outputPath);
|
|
61
|
+
const localMountPath = path.resolve(outputDir, 'mount');
|
|
62
|
+
if (!fs.existsSync(localMountPath)) {
|
|
63
|
+
fs.mkdirSync(localMountPath, { recursive: true });
|
|
64
|
+
}
|
|
65
|
+
return content.replace(/\/mnt\/data/g, localMountPath);
|
|
66
|
+
}
|
|
67
|
+
|
|
52
68
|
/**
|
|
53
69
|
* Resolve output path for env file
|
|
54
70
|
* @param {string} rawOutputPath - Raw output path from application config
|
|
@@ -100,7 +116,8 @@ async function writeEnvOutputForReload(outputPath, runEnvPath) {
|
|
|
100
116
|
*/
|
|
101
117
|
async function writeEnvOutputForLocal(appName, outputPath) {
|
|
102
118
|
const { generateEnvContent, parseEnvContentToMap, mergeEnvMapIntoContent } = require('../core/secrets');
|
|
103
|
-
|
|
119
|
+
let localContent = await generateEnvContent(appName, null, 'local', false);
|
|
120
|
+
localContent = substituteMntDataForLocal(localContent, outputPath);
|
|
104
121
|
let toWrite = localContent;
|
|
105
122
|
if (fs.existsSync(outputPath)) {
|
|
106
123
|
const existingContent = await fsp.readFile(outputPath, 'utf8');
|
|
@@ -229,7 +246,8 @@ async function patchEnvContentForLocal(envContent, variables) {
|
|
|
229
246
|
*/
|
|
230
247
|
async function writeLocalEnvToOutputPath(outputPath, appName, secretsPath, envOutputPathLabel) {
|
|
231
248
|
const { generateEnvContent, parseEnvContentToMap, mergeEnvMapIntoContent } = require('../core/secrets');
|
|
232
|
-
|
|
249
|
+
let localEnvContent = await generateEnvContent(appName, secretsPath, 'local', false);
|
|
250
|
+
localEnvContent = substituteMntDataForLocal(localEnvContent, outputPath);
|
|
233
251
|
let toWrite = localEnvContent;
|
|
234
252
|
if (fs.existsSync(outputPath)) {
|
|
235
253
|
const existingContent = fs.readFileSync(outputPath, 'utf8');
|
|
@@ -250,7 +268,8 @@ async function writeLocalEnvToOutputPath(outputPath, appName, secretsPath, envOu
|
|
|
250
268
|
*/
|
|
251
269
|
async function writePatchedEnvToOutputPath(envPath, outputPath, variables, envOutputPathLabel) {
|
|
252
270
|
const envContent = fs.readFileSync(envPath, 'utf8');
|
|
253
|
-
|
|
271
|
+
let patchedContent = await patchEnvContentForLocal(envContent, variables);
|
|
272
|
+
patchedContent = substituteMntDataForLocal(patchedContent, outputPath);
|
|
254
273
|
fs.writeFileSync(outputPath, patchedContent, { mode: 0o600 });
|
|
255
274
|
logger.log(chalk.green(`✓ Copied .env to: ${envOutputPathLabel}`));
|
|
256
275
|
}
|
|
@@ -292,6 +311,7 @@ async function processEnvVariables(envPath, variablesPath, appName, secretsPath)
|
|
|
292
311
|
module.exports = {
|
|
293
312
|
processEnvVariables,
|
|
294
313
|
resolveEnvOutputPath,
|
|
314
|
+
substituteMntDataForLocal,
|
|
295
315
|
writeEnvOutputForReload,
|
|
296
316
|
writeEnvOutputForLocal
|
|
297
317
|
};
|
|
@@ -30,42 +30,92 @@ function formatDisplayName(key) {
|
|
|
30
30
|
.join(' ');
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Derives suffix from datasource key for filename generation
|
|
35
|
+
* @param {string} key - Datasource key
|
|
36
|
+
* @param {string} systemKey - System key
|
|
37
|
+
* @param {string} entityType - Fallback entity type
|
|
38
|
+
* @returns {string} Suffix segment
|
|
39
|
+
*/
|
|
40
|
+
function getDatasourceKeySuffix(key, systemKey, entityType) {
|
|
41
|
+
if (key.startsWith(`${systemKey}-deploy-`)) {
|
|
42
|
+
return key.slice(`${systemKey}-deploy-`.length);
|
|
43
|
+
}
|
|
44
|
+
if (systemKey && key.startsWith(`${systemKey}-`)) {
|
|
45
|
+
return key.slice(systemKey.length + 1);
|
|
46
|
+
}
|
|
47
|
+
if (key) {
|
|
48
|
+
return key;
|
|
49
|
+
}
|
|
50
|
+
return entityType;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Normalizes a single datasource entry for template use
|
|
55
|
+
* @param {Object} datasource - Datasource object
|
|
56
|
+
* @param {number} index - Index in array
|
|
57
|
+
* @param {string} systemKey - System key for filename generation
|
|
58
|
+
* @param {string} ext - File extension (e.g. '.json', '.yaml')
|
|
59
|
+
* @returns {{entityType: string, displayName: string, fileName: string, datasourceKey: string}} Normalized entry
|
|
60
|
+
*/
|
|
61
|
+
function normalizeOneDatasource(datasource, index, systemKey, ext) {
|
|
62
|
+
const entityType = datasource.entityType ||
|
|
63
|
+
datasource.entityKey ||
|
|
64
|
+
datasource.key?.split('-').pop() ||
|
|
65
|
+
`entity${index + 1}`;
|
|
66
|
+
const displayName = datasource.displayName ||
|
|
67
|
+
datasource.name ||
|
|
68
|
+
`Datasource ${index + 1}`;
|
|
69
|
+
const key = datasource.key || '';
|
|
70
|
+
const suffix = getDatasourceKeySuffix(key, systemKey, entityType);
|
|
71
|
+
const datasourceKey = key || (systemKey ? `${systemKey}-${suffix}` : suffix);
|
|
72
|
+
const fileName = datasource.fileName || datasource.file ||
|
|
73
|
+
(systemKey ? `${systemKey}-datasource-${suffix}${ext}` : `${suffix}${ext}`);
|
|
74
|
+
return { entityType, displayName, fileName, datasourceKey };
|
|
75
|
+
}
|
|
76
|
+
|
|
33
77
|
/**
|
|
34
78
|
* Normalizes datasource entries for template use
|
|
35
79
|
* @param {Array} datasources - Datasource objects
|
|
36
80
|
* @param {string} systemKey - System key for filename generation
|
|
37
|
-
* @
|
|
81
|
+
* @param {string} [fileExt='.json'] - File extension for generated filenames (e.g. '.json', '.yaml')
|
|
82
|
+
* @returns {Array<{entityType: string, displayName: string, fileName: string, datasourceKey: string}>} Normalized entries
|
|
38
83
|
*/
|
|
39
|
-
function normalizeDatasources(datasources, systemKey) {
|
|
84
|
+
function normalizeDatasources(datasources, systemKey, fileExt = '.json') {
|
|
40
85
|
if (!Array.isArray(datasources)) {
|
|
41
86
|
return [];
|
|
42
87
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
|
|
88
|
+
const ext = fileExt && fileExt.startsWith('.') ? fileExt : `.${fileExt || 'json'}`;
|
|
89
|
+
return datasources.map((datasource, index) =>
|
|
90
|
+
normalizeOneDatasource(datasource, index, systemKey, ext)
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Builds secret path entries for README "Secrets" section per auth type.
|
|
96
|
+
* Path is the key for `aifabrix secret set <key> <value>` (no kv:// prefix; key format systemKey/secretKey in camelCase).
|
|
97
|
+
* @param {string} systemKey - System key
|
|
98
|
+
* @param {string} [authType] - Authentication type (oauth2, aad, apikey, basic, queryParam, hmac, bearer, token, none)
|
|
99
|
+
* @returns {Array<{path: string, description: string}>} secretPaths for template (path = key for secret set, no kv://)
|
|
100
|
+
*/
|
|
101
|
+
function buildSecretPaths(systemKey, authType) {
|
|
102
|
+
if (!systemKey || typeof systemKey !== 'string') return [];
|
|
103
|
+
const t = (authType && typeof authType === 'string') ? authType.toLowerCase() : 'apikey';
|
|
104
|
+
const map = {
|
|
105
|
+
oauth2: [{ path: `${systemKey}/clientId`, description: 'Client ID' }, { path: `${systemKey}/clientSecret`, description: 'Client Secret' }],
|
|
106
|
+
oauth: [{ path: `${systemKey}/clientId`, description: 'Client ID' }, { path: `${systemKey}/clientSecret`, description: 'Client Secret' }],
|
|
107
|
+
aad: [{ path: `${systemKey}/clientId`, description: 'Client ID' }, { path: `${systemKey}/clientSecret`, description: 'Client Secret' }],
|
|
108
|
+
apikey: [{ path: `${systemKey}/apiKey`, description: 'API Key' }],
|
|
109
|
+
apiKey: [{ path: `${systemKey}/apiKey`, description: 'API Key' }],
|
|
110
|
+
basic: [{ path: `${systemKey}/username`, description: 'Username' }, { path: `${systemKey}/password`, description: 'Password' }],
|
|
111
|
+
queryparam: [{ path: `${systemKey}/paramValue`, description: 'Query parameter value' }],
|
|
112
|
+
hmac: [{ path: `${systemKey}/signingSecret`, description: 'Signing secret' }],
|
|
113
|
+
bearer: [{ path: `${systemKey}/bearerToken`, description: 'Bearer token' }],
|
|
114
|
+
token: [{ path: `${systemKey}/bearerToken`, description: 'Bearer token' }],
|
|
115
|
+
oidc: [],
|
|
116
|
+
none: []
|
|
117
|
+
};
|
|
118
|
+
return map[t] || map.apikey;
|
|
69
119
|
}
|
|
70
120
|
|
|
71
121
|
/**
|
|
@@ -78,6 +128,9 @@ function normalizeDatasources(datasources, systemKey) {
|
|
|
78
128
|
* @param {string} [params.displayName] - Display name
|
|
79
129
|
* @param {string} [params.description] - Description
|
|
80
130
|
* @param {Array} [params.datasources] - Datasource objects
|
|
131
|
+
* @param {string} [params.fileExt] - File extension for config files (e.g. '.json', '.yaml'); default '.json'
|
|
132
|
+
* @param {string} [params.authType] - Authentication type for Secrets section (oauth2, aad, apikey, basic, etc.)
|
|
133
|
+
* @param {Object} [params.authentication] - Full authentication object (authType used if authType not set)
|
|
81
134
|
* @returns {Object} Template context
|
|
82
135
|
*/
|
|
83
136
|
function buildExternalReadmeContext(params = {}) {
|
|
@@ -86,7 +139,10 @@ function buildExternalReadmeContext(params = {}) {
|
|
|
86
139
|
const displayName = params.displayName || formatDisplayName(systemKey);
|
|
87
140
|
const description = params.description || `External system integration for ${systemKey}`;
|
|
88
141
|
const systemType = params.systemType || 'openapi';
|
|
89
|
-
const
|
|
142
|
+
const fileExt = params.fileExt !== undefined ? params.fileExt : '.json';
|
|
143
|
+
const datasources = normalizeDatasources(params.datasources, systemKey, fileExt);
|
|
144
|
+
const authType = params.authType || params.authentication?.type || params.authentication?.method || params.authentication?.authType;
|
|
145
|
+
const secretPaths = buildSecretPaths(systemKey, authType);
|
|
90
146
|
|
|
91
147
|
return {
|
|
92
148
|
appName,
|
|
@@ -94,8 +150,11 @@ function buildExternalReadmeContext(params = {}) {
|
|
|
94
150
|
displayName,
|
|
95
151
|
description,
|
|
96
152
|
systemType,
|
|
153
|
+
fileExt: fileExt && fileExt.startsWith('.') ? fileExt : `.${fileExt || 'json'}`,
|
|
97
154
|
datasourceCount: datasources.length,
|
|
98
|
-
datasources
|
|
155
|
+
hasDatasources: datasources.length > 0,
|
|
156
|
+
datasources,
|
|
157
|
+
secretPaths
|
|
99
158
|
};
|
|
100
159
|
}
|
|
101
160
|
|
|
@@ -241,8 +241,66 @@ function displayIntegrationTestResults(results, verbose = false) {
|
|
|
241
241
|
}
|
|
242
242
|
}
|
|
243
243
|
|
|
244
|
+
/**
|
|
245
|
+
* Displays E2E test results (steps: config, credential, sync, data, cip).
|
|
246
|
+
* Supports sync response (data.steps only), final poll (data.steps + data.success), and running poll
|
|
247
|
+
* (data.completedActions, no data.steps yet). When status is present (async flow), shows it.
|
|
248
|
+
*
|
|
249
|
+
* @param {Object} data - E2E response or poll data
|
|
250
|
+
* @param {string} [data.status] - Optional status: 'running' | 'completed' | 'failed' (async flow)
|
|
251
|
+
* @param {Object[]} [data.steps] - Per-step results (final state)
|
|
252
|
+
* @param {Object[]} [data.completedActions] - Steps completed so far (running state when steps absent)
|
|
253
|
+
* @param {boolean} [data.success] - Overall success (final state)
|
|
254
|
+
* @param {string} [data.error] - Error message when failed
|
|
255
|
+
* @param {boolean} [verbose] - Show detailed output
|
|
256
|
+
*/
|
|
257
|
+
/* eslint-disable max-statements,complexity -- Step iteration and status display */
|
|
258
|
+
function displayE2EResults(data, verbose = false) {
|
|
259
|
+
logger.log(chalk.blue('\n📊 E2E Test Results\n'));
|
|
260
|
+
if (data.status) {
|
|
261
|
+
const statusLabel = data.status === 'running'
|
|
262
|
+
? chalk.yellow('running')
|
|
263
|
+
: data.status === 'completed'
|
|
264
|
+
? chalk.green('completed')
|
|
265
|
+
: data.status === 'failed'
|
|
266
|
+
? chalk.red('failed')
|
|
267
|
+
: data.status;
|
|
268
|
+
logger.log(`Status: ${statusLabel}`);
|
|
269
|
+
}
|
|
270
|
+
const steps = data.steps || data.completedActions || [];
|
|
271
|
+
if (steps.length === 0) {
|
|
272
|
+
if (data.success === false) {
|
|
273
|
+
logger.log(chalk.red('✗ E2E test failed'));
|
|
274
|
+
if (data.error) logger.log(chalk.red(` Error: ${data.error}`));
|
|
275
|
+
} else if (data.status === 'running') {
|
|
276
|
+
logger.log(chalk.gray(' No steps completed yet'));
|
|
277
|
+
} else {
|
|
278
|
+
logger.log(chalk.yellow('No step results returned'));
|
|
279
|
+
}
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
const isRunning = data.status === 'running' && !data.steps;
|
|
283
|
+
if (isRunning && verbose) {
|
|
284
|
+
logger.log(chalk.gray(` (${steps.length} step(s) completed so far)`));
|
|
285
|
+
}
|
|
286
|
+
for (const step of steps) {
|
|
287
|
+
const name = step.name || step.step || 'unknown';
|
|
288
|
+
const ok = step.success !== false && !step.error;
|
|
289
|
+
logger.log(` ${ok ? chalk.green('✓') : chalk.red('✗')} ${name}`);
|
|
290
|
+
if (!ok && (step.error || step.message)) logger.log(chalk.red(` ${step.error || step.message}`));
|
|
291
|
+
if (verbose && step.message && ok) logger.log(chalk.gray(` ${step.message}`));
|
|
292
|
+
}
|
|
293
|
+
if (isRunning) {
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
const allPassed = steps.every(s => s.success !== false && !s.error);
|
|
297
|
+
logger.log(allPassed ? chalk.green('\n✅ E2E test passed!') : chalk.red('\n❌ E2E test failed'));
|
|
298
|
+
}
|
|
299
|
+
|
|
244
300
|
module.exports = {
|
|
245
301
|
displayTestResults,
|
|
246
|
-
displayIntegrationTestResults
|
|
302
|
+
displayIntegrationTestResults,
|
|
303
|
+
displayE2EResults,
|
|
304
|
+
displayDatasourceIntegrationResult
|
|
247
305
|
};
|
|
248
306
|
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
const fs = require('fs').promises;
|
|
13
13
|
const path = require('path');
|
|
14
14
|
const { testDatasourceViaPipeline } = require('../api/pipeline.api');
|
|
15
|
-
|
|
15
|
+
|
|
16
|
+
/** Pipeline test endpoints accept client credentials; do not enforce Bearer-only */
|
|
16
17
|
|
|
17
18
|
/**
|
|
18
19
|
* Retry API call with exponential backoff
|
|
@@ -40,26 +41,31 @@ async function retryApiCall(fn, maxRetries = 3, backoffMs = 1000) {
|
|
|
40
41
|
}
|
|
41
42
|
|
|
42
43
|
/**
|
|
43
|
-
* Calls pipeline test endpoint using centralized API client
|
|
44
|
+
* Calls pipeline test endpoint using centralized API client.
|
|
45
|
+
* Pipeline test accepts Bearer, API_KEY, or client credentials (x-client-id/x-client-secret) for CI/CD.
|
|
44
46
|
* @async
|
|
45
47
|
* @param {Object} params - Function parameters
|
|
46
48
|
* @param {string} params.systemKey - System key
|
|
47
49
|
* @param {string} params.datasourceKey - Datasource key
|
|
48
50
|
* @param {Object} params.payloadTemplate - Test payload template
|
|
49
51
|
* @param {string} params.dataplaneUrl - Dataplane URL
|
|
50
|
-
* @param {Object} params.authConfig - Authentication configuration
|
|
52
|
+
* @param {Object} params.authConfig - Authentication configuration (token or client credentials)
|
|
51
53
|
* @param {number} [params.timeout] - Request timeout in milliseconds (default: 30000)
|
|
54
|
+
* @param {boolean} [params.includeDebug] - Include debug output in response
|
|
52
55
|
* @returns {Promise<Object>} Test response
|
|
53
56
|
*/
|
|
54
|
-
async function callPipelineTestEndpoint({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout = 30000 }) {
|
|
55
|
-
|
|
57
|
+
async function callPipelineTestEndpoint({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout = 30000, includeDebug = false }) {
|
|
58
|
+
const testData = { payloadTemplate };
|
|
59
|
+
if (includeDebug) {
|
|
60
|
+
testData.includeDebug = true;
|
|
61
|
+
}
|
|
56
62
|
const response = await retryApiCall(async() => {
|
|
57
63
|
return await testDatasourceViaPipeline({
|
|
58
64
|
dataplaneUrl,
|
|
59
65
|
systemKey,
|
|
60
66
|
datasourceKey,
|
|
61
67
|
authConfig,
|
|
62
|
-
testData
|
|
68
|
+
testData,
|
|
63
69
|
options: { timeout }
|
|
64
70
|
});
|
|
65
71
|
});
|
|
@@ -67,6 +73,11 @@ async function callPipelineTestEndpoint({ systemKey, datasourceKey, payloadTempl
|
|
|
67
73
|
if (!response.success || !response.data) {
|
|
68
74
|
throw new Error(`Test endpoint failed: ${response.error || response.formattedError || 'Unknown error'}`);
|
|
69
75
|
}
|
|
76
|
+
// When 200 with success: false in body, pass through; caller interprets via data.success
|
|
77
|
+
if (response.data?.success === false) {
|
|
78
|
+
const errMsg = response.data?.error || response.data?.formattedError || 'Test failed';
|
|
79
|
+
throw new Error(`Test endpoint failed: ${errMsg}`);
|
|
80
|
+
}
|
|
70
81
|
|
|
71
82
|
return response.data.data || response.data;
|
|
72
83
|
}
|
|
@@ -114,16 +125,18 @@ function determinePayloadTemplate(datasource, datasourceKey, customPayload) {
|
|
|
114
125
|
* @param {string} params.dataplaneUrl - Dataplane URL
|
|
115
126
|
* @param {Object} params.authConfig - Authentication configuration
|
|
116
127
|
* @param {number} params.timeout - Request timeout
|
|
128
|
+
* @param {boolean} [params.includeDebug] - Include debug in response
|
|
117
129
|
* @returns {Promise<Object>} Test result
|
|
118
130
|
*/
|
|
119
|
-
async function testSingleDatasource({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout }) {
|
|
131
|
+
async function testSingleDatasource({ systemKey, datasourceKey, payloadTemplate, dataplaneUrl, authConfig, timeout, includeDebug = false }) {
|
|
120
132
|
const testResponse = await callPipelineTestEndpoint({
|
|
121
133
|
systemKey,
|
|
122
134
|
datasourceKey,
|
|
123
135
|
payloadTemplate,
|
|
124
136
|
dataplaneUrl,
|
|
125
137
|
authConfig,
|
|
126
|
-
timeout
|
|
138
|
+
timeout,
|
|
139
|
+
includeDebug
|
|
127
140
|
});
|
|
128
141
|
|
|
129
142
|
return {
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
const Ajv = require('ajv');
|
|
12
|
+
const addFormats = require('ajv-formats');
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
15
|
* Validates field mapping expression syntax (pipe-based DSL)
|
|
@@ -263,6 +264,7 @@ function validateMetadataSchema(datasource, testPayload) {
|
|
|
263
264
|
|
|
264
265
|
try {
|
|
265
266
|
const ajv = new Ajv({ allErrors: true, strict: false });
|
|
267
|
+
addFormats(ajv);
|
|
266
268
|
const validate = ajv.compile(datasource.metadataSchema);
|
|
267
269
|
const valid = validate(payloadTemplate);
|
|
268
270
|
|
|
@@ -319,6 +321,7 @@ function validateAgainstSchema(data, schema) {
|
|
|
319
321
|
allowUnionTypes: true,
|
|
320
322
|
validateSchema: false
|
|
321
323
|
});
|
|
324
|
+
addFormats(ajv);
|
|
322
325
|
// Remove $schema for draft-2020-12 to avoid AJV issues
|
|
323
326
|
const schemaCopy = { ...schema };
|
|
324
327
|
if (schemaCopy.$schema && schemaCopy.$schema.includes('2020-12')) {
|
package/lib/utils/file-upload.js
CHANGED
|
@@ -1,29 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview File upload utilities for multipart/form-data requests
|
|
3
|
+
* All API calls go via ApiClient (lib/api/index.js); no duplicate auth logic.
|
|
3
4
|
* @author AI Fabrix Team
|
|
4
5
|
* @version 2.0.0
|
|
5
6
|
*/
|
|
6
7
|
|
|
7
8
|
const fs = require('fs').promises;
|
|
8
9
|
const path = require('path');
|
|
9
|
-
const {
|
|
10
|
+
const { ApiClient } = require('../api');
|
|
10
11
|
|
|
11
|
-
/**
|
|
12
|
-
* Upload a file using multipart/form-data
|
|
13
|
-
* @async
|
|
14
|
-
* @function uploadFile
|
|
15
|
-
* @param {string} url - API endpoint URL
|
|
16
|
-
* @param {string} filePath - Path to file to upload
|
|
17
|
-
* @param {string} fieldName - Form field name for the file (default: 'file')
|
|
18
|
-
* @param {Object} [authConfig] - Authentication configuration
|
|
19
|
-
* @param {string} [authConfig.type] - Auth type ('bearer' | 'client-credentials')
|
|
20
|
-
* @param {string} [authConfig.token] - Bearer token
|
|
21
|
-
* @param {string} [authConfig.clientId] - Client ID
|
|
22
|
-
* @param {string} [authConfig.clientSecret] - Client secret
|
|
23
|
-
* @param {Object} [additionalFields] - Additional form fields to include
|
|
24
|
-
* @returns {Promise<Object>} API response
|
|
25
|
-
* @throws {Error} If file upload fails
|
|
26
|
-
*/
|
|
27
12
|
/**
|
|
28
13
|
* Validates file exists
|
|
29
14
|
* @async
|
|
@@ -63,47 +48,32 @@ async function buildFormData(filePath, fieldName, additionalFields) {
|
|
|
63
48
|
}
|
|
64
49
|
|
|
65
50
|
/**
|
|
66
|
-
*
|
|
67
|
-
* @
|
|
68
|
-
* @
|
|
69
|
-
* @
|
|
51
|
+
* Upload a file using multipart/form-data via ApiClient (single place for auth and API calls).
|
|
52
|
+
* @async
|
|
53
|
+
* @function uploadFile
|
|
54
|
+
* @param {string} url - Full API endpoint URL (e.g. https://dataplane.example.com/api/v1/wizard/parse-openapi)
|
|
55
|
+
* @param {string} filePath - Path to file to upload
|
|
56
|
+
* @param {string} fieldName - Form field name for the file (default: 'file')
|
|
57
|
+
* @param {Object} [authConfig] - Authentication configuration (token-only for app endpoints)
|
|
58
|
+
* @param {string} [authConfig.type] - Auth type ('bearer' | 'client-token')
|
|
59
|
+
* @param {string} [authConfig.token] - Token (Bearer user token or x-client-token application token)
|
|
60
|
+
* @param {Object} [additionalFields] - Additional form fields to include
|
|
61
|
+
* @returns {Promise<Object>} API response
|
|
62
|
+
* @throws {Error} If file upload fails
|
|
70
63
|
*/
|
|
71
|
-
function buildAuthHeaders(authConfig) {
|
|
72
|
-
const headers = {};
|
|
73
|
-
if (authConfig.type === 'bearer' && authConfig.token) {
|
|
74
|
-
headers['Authorization'] = `Bearer ${authConfig.token}`;
|
|
75
|
-
} else if (authConfig.type === 'client-credentials') {
|
|
76
|
-
if (authConfig.clientId) {
|
|
77
|
-
headers['x-client-id'] = authConfig.clientId;
|
|
78
|
-
}
|
|
79
|
-
if (authConfig.clientSecret) {
|
|
80
|
-
headers['x-client-secret'] = authConfig.clientSecret;
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
return headers;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
64
|
async function uploadFile(url, filePath, fieldName = 'file', authConfig = {}, additionalFields = {}) {
|
|
87
65
|
await validateFileExists(filePath);
|
|
88
66
|
|
|
89
|
-
const
|
|
90
|
-
const
|
|
91
|
-
|
|
92
|
-
const options = {
|
|
93
|
-
method: 'POST',
|
|
94
|
-
headers,
|
|
95
|
-
body: formData
|
|
96
|
-
};
|
|
67
|
+
const parsed = new URL(url);
|
|
68
|
+
const baseUrl = parsed.origin;
|
|
69
|
+
const endpointPath = parsed.pathname + parsed.search;
|
|
97
70
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
return await authenticatedApiCall(url, options, authConfig.token);
|
|
101
|
-
}
|
|
71
|
+
const formData = await buildFormData(filePath, fieldName, additionalFields);
|
|
72
|
+
const client = new ApiClient(baseUrl, authConfig);
|
|
102
73
|
|
|
103
|
-
return await
|
|
74
|
+
return await client.postFormData(endpointPath, formData);
|
|
104
75
|
}
|
|
105
76
|
|
|
106
77
|
module.exports = {
|
|
107
78
|
uploadFile
|
|
108
79
|
};
|
|
109
|
-
|
|
@@ -97,6 +97,7 @@ const CATEGORIES = [
|
|
|
97
97
|
{ name: 'download', term: 'download <system-key>' },
|
|
98
98
|
{ name: 'upload', term: 'upload <system-key>' },
|
|
99
99
|
{ name: 'delete', term: 'delete <system-key>' },
|
|
100
|
+
{ name: 'repair', term: 'repair <app>' },
|
|
100
101
|
{ name: 'test', term: 'test <app>' },
|
|
101
102
|
{ name: 'test-integration', term: 'test-integration <app>' }
|
|
102
103
|
]
|