@aifabrix/builder 2.45.0 → 2.45.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursor/rules/docs-rules.mdc +3 -1
- package/integration/hubspot-test/test-artifacts/wizard-hubspot-credential-real.yaml +1 -1
- package/integration/hubspot-test/test-artifacts/wizard-hubspot-env-vars.yaml +1 -1
- package/integration/hubspot-test/wizard-hubspot-e2e.yaml +1 -1
- package/jest.projects.js +44 -1
- package/lib/api/agent-metadata-validation.api.js +93 -0
- package/lib/api/audit-api-helpers.js +68 -0
- package/lib/api/audit.api.js +119 -0
- package/lib/api/auth-cache.api.js +44 -0
- package/lib/api/certificates.api.js +4 -4
- package/lib/api/credential.api.js +2 -2
- package/lib/api/dataplane-health.api.js +137 -0
- package/lib/api/dataplane-sync.api.js +27 -0
- package/lib/api/datasources-core.api.js +21 -18
- package/lib/api/datasources-extended.api.js +22 -22
- package/lib/api/dimensions.api.js +3 -0
- package/lib/api/external-systems.api.js +13 -13
- package/lib/api/governance-scenarios.api.js +39 -0
- package/lib/api/groups.api.js +79 -0
- package/lib/api/index.js +45 -2
- package/lib/api/pipeline.api.js +4 -4
- package/lib/api/protection.api.js +157 -0
- package/lib/api/records-bulk.api.js +30 -0
- package/lib/api/records-search-parse.js +49 -0
- package/lib/api/records-search.api.js +29 -0
- package/lib/api/types/agent-metadata-validation.types.js +67 -0
- package/lib/api/types/audit.types.js +72 -0
- package/lib/api/types/dataplane-health.types.js +27 -0
- package/lib/api/types/dimensions.types.js +2 -0
- package/lib/api/types/governance-scenarios.types.js +39 -0
- package/lib/api/types/protection.types.js +69 -0
- package/lib/api/types/records-bulk.types.js +35 -0
- package/lib/api/types/records-search.types.js +28 -0
- package/lib/api/user-groups.api.js +50 -0
- package/lib/api/users.api.js +95 -0
- package/lib/api/validation-run.api.js +3 -3
- package/lib/api/validation-runner.js +3 -3
- package/lib/api/wizard-platform.api.js +3 -3
- package/lib/api/wizard.api.js +21 -21
- package/lib/app/deploy.js +34 -19
- package/lib/cli/index.js +4 -2
- package/lib/cli/infra-guided.js +22 -3
- package/lib/cli/setup-app.help.js +50 -2
- package/lib/cli/setup-app.js +33 -24
- package/lib/cli/setup-app.test-commands.js +80 -21
- package/lib/cli/setup-auth.js +30 -3
- package/lib/cli/setup-external-system.js +37 -3
- package/lib/cli/setup-utility-help-after.js +45 -0
- package/lib/cli/setup-utility-resolve.js +2 -2
- package/lib/cli/setup-utility.js +99 -81
- package/lib/commands/auth-status-dataplane-version.js +231 -0
- package/lib/commands/auth-status-display.js +94 -0
- package/lib/commands/auth-status.js +70 -6
- package/lib/commands/credential-push.js +10 -5
- package/lib/commands/datasource-load-export-cli.js +178 -0
- package/lib/commands/datasource-test-trust-cli.js +114 -0
- package/lib/commands/datasource-unified-test-cli.js +25 -17
- package/lib/commands/datasource-unified-test-cli.options.js +18 -3
- package/lib/commands/datasource-unified-test-e2e-cli-helpers.js +5 -5
- package/lib/commands/datasource-verify-audit-cli.js +96 -0
- package/lib/commands/datasource.js +68 -1
- package/lib/commands/dimension-value.js +43 -13
- package/lib/commands/dimension.js +38 -7
- package/lib/commands/identity-apply-sync.js +131 -0
- package/lib/commands/identity-group-cli.js +146 -0
- package/lib/commands/identity-membership-role-cache.js +175 -0
- package/lib/commands/identity-shared.js +94 -0
- package/lib/commands/identity-user-cli.js +158 -0
- package/lib/commands/identity.js +43 -0
- package/lib/commands/login.js +46 -16
- package/lib/commands/protection-cli-leaves.js +286 -0
- package/lib/commands/protection.js +50 -0
- package/lib/commands/repair.js +13 -3
- package/lib/commands/test-e2e-external.js +40 -9
- package/lib/commands/test-governance-cli-options.js +47 -0
- package/lib/commands/test-governance-command-action.js +55 -0
- package/lib/commands/test-governance-external.js +88 -0
- package/lib/commands/test-trust-cli-options.js +32 -0
- package/lib/commands/test-trust-command-action.js +118 -0
- package/lib/commands/test-trust-external.js +127 -0
- package/lib/commands/upload.js +34 -13
- package/lib/commands/wizard-core.js +3 -0
- package/lib/core/config-device-dataplane.js +185 -0
- package/lib/datasource/agent-trust-debug-log.js +45 -0
- package/lib/datasource/agent-trust-fetch.js +115 -0
- package/lib/datasource/agent-trust-map.js +101 -0
- package/lib/datasource/agent-trust-mock.js +44 -0
- package/lib/datasource/agent-trust-run.js +93 -0
- package/lib/datasource/agent-trust-timeout.js +25 -0
- package/lib/datasource/audit-evidence-display.js +80 -0
- package/lib/datasource/audit-evidence-extract-debug.js +103 -0
- package/lib/datasource/audit-evidence-extract.js +177 -0
- package/lib/datasource/audit-evidence-matrix-rows.js +158 -0
- package/lib/datasource/audit-evidence-operation-aliases.js +63 -0
- package/lib/datasource/audit-evidence-poll.js +109 -0
- package/lib/datasource/audit-evidence-run.js +177 -0
- package/lib/datasource/audit-evidence-verify-helpers.js +167 -0
- package/lib/datasource/audit-evidence-verify-matrix.js +177 -0
- package/lib/datasource/audit-evidence-verify.js +108 -0
- package/lib/datasource/bulk-loader-run-batches.js +88 -0
- package/lib/datasource/bulk-loader-service-retry.js +52 -0
- package/lib/datasource/bulk-loader-service.js +127 -0
- package/lib/datasource/capability/run-capability-dimension.js +2 -1
- package/lib/datasource/datasource-exporter-resolve.js +82 -0
- package/lib/datasource/datasource-exporter-service.js +144 -0
- package/lib/datasource/datasource-identifier.js +148 -0
- package/lib/datasource/datasource-load-export-context.js +92 -0
- package/lib/datasource/integration-context.js +6 -4
- package/lib/datasource/local-data-paths.js +145 -0
- package/lib/datasource/log-cleaner.js +241 -0
- package/lib/datasource/log-viewer.js +111 -6
- package/lib/datasource/record-file-parser.js +85 -0
- package/lib/datasource/record-mapper.js +129 -0
- package/lib/datasource/resolve-app.js +74 -55
- package/lib/datasource/test-e2e.js +26 -2
- package/lib/datasource/unified-validation-run-resolve.js +45 -12
- package/lib/datasource/unified-validation-run.js +43 -17
- package/lib/datasource/validate.js +37 -20
- package/lib/external-system/test-system-level.js +25 -4
- package/lib/external-system/test.js +3 -1
- package/lib/governance/governance-pack-loader.js +70 -0
- package/lib/governance/governance-run-display.js +119 -0
- package/lib/identity/identity-apply-core.js +108 -0
- package/lib/identity/identity-apply-phases.js +244 -0
- package/lib/identity/identity-apply-service.js +77 -0
- package/lib/identity/identity-csv-parser.js +114 -0
- package/lib/internal/fs-real-sync.js +29 -1
- package/lib/protection/auth-context.js +45 -0
- package/lib/protection/convert-batch.js +90 -0
- package/lib/protection/load.js +41 -0
- package/lib/protection/paths.js +158 -0
- package/lib/protection/preflight-datasource-ready.js +46 -0
- package/lib/protection/protection-create-probe-lines.js +142 -0
- package/lib/protection/protection-create-probes.js +95 -0
- package/lib/protection/protection-create-scaffold.js +281 -0
- package/lib/protection/protection-display-helpers.js +445 -0
- package/lib/protection/protection-display.js +215 -0
- package/lib/protection/protection-preset-registry.js +97 -0
- package/lib/protection/report-exit.js +62 -0
- package/lib/protection/resolve.js +202 -0
- package/lib/protection/run-commands-validate.js +130 -0
- package/lib/protection/run-commands.js +216 -0
- package/lib/protection/run-protection-create-helpers.js +134 -0
- package/lib/protection/run-protection-create-probes-flow.js +43 -0
- package/lib/protection/run-protection-create.js +191 -0
- package/lib/protection/scope.js +22 -0
- package/lib/protection/sync-after-upload.js +64 -0
- package/lib/protection/upload-batch.js +127 -0
- package/lib/protection/validate-batch.js +113 -0
- package/lib/protection/validate-local.js +65 -0
- package/lib/resolvers/dimension-file.js +29 -4
- package/lib/schema/external-datasource.schema.json +47 -13
- package/lib/schema/flag-map-validation-run.json +2 -2
- package/lib/schema/protection.schema.json +374 -0
- package/lib/schema/type/credential-field-registry.json +636 -0
- package/lib/schema/wizard-config.schema.json +1 -1
- package/lib/utils/agent-trust-run-display.js +231 -0
- package/lib/utils/agent-trust-run-exit.js +59 -0
- package/lib/utils/agent-trust-wait-ui.js +94 -0
- package/lib/utils/cli-exit-codes.js +30 -0
- package/lib/utils/cli-sync-options.js +33 -0
- package/lib/utils/credential-push-ui.js +90 -0
- package/lib/utils/dataplane-cli-version-gate.js +202 -0
- package/lib/utils/dataplane-cli-version-help.js +74 -0
- package/lib/utils/datasource-export-display.js +137 -0
- package/lib/utils/datasource-load-display.js +174 -0
- package/lib/utils/datasource-load-export-ui.js +120 -0
- package/lib/utils/datasource-test-run-capability-scope.js +3 -3
- package/lib/utils/datasource-test-run-display.js +3 -3
- package/lib/utils/datasource-test-run-tty-log.js +5 -1
- package/lib/utils/help-builder.js +30 -24
- package/lib/utils/manifest-source-emit.js +22 -0
- package/lib/utils/semver-compare.js +127 -0
- package/lib/utils/upload-pipeline-ui.js +91 -0
- package/lib/utils/validation-poll-ui.js +14 -7
- package/lib/validation/wizard-config-validator.js +58 -19
- package/package.json +1 -1
- package/templates/applications/dataplane/env.template +13 -1
- package/templates/applications/dataplane/rbac.yaml +4 -0
- package/templates/external-system/README.md.hbs +56 -8
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Map fixture payloads to canonical bulk sync records (plan 144)
|
|
3
|
+
* @author AI Fabrix Team
|
|
4
|
+
* @version 1.0.0
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
'use strict';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* @param {Object} obj
|
|
11
|
+
* @returns {boolean}
|
|
12
|
+
*/
|
|
13
|
+
function isCanonicalBulkRecord(obj) {
|
|
14
|
+
return (
|
|
15
|
+
obj &&
|
|
16
|
+
typeof obj === 'object' &&
|
|
17
|
+
typeof obj.key === 'string' &&
|
|
18
|
+
obj.key.trim().length > 0 &&
|
|
19
|
+
typeof obj.recordType === 'string' &&
|
|
20
|
+
obj.recordType.trim().length > 0 &&
|
|
21
|
+
obj.metadata !== undefined &&
|
|
22
|
+
obj.metadata !== null &&
|
|
23
|
+
typeof obj.metadata === 'object' &&
|
|
24
|
+
!Array.isArray(obj.metadata)
|
|
25
|
+
);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* @param {string[]} primaryKeyFields
|
|
30
|
+
* @param {Object} payload
|
|
31
|
+
* @returns {string}
|
|
32
|
+
*/
|
|
33
|
+
function buildKeyFromPrimaryKey(primaryKeyFields, payload) {
|
|
34
|
+
if (!primaryKeyFields.length) {
|
|
35
|
+
throw new Error('Datasource primaryKey is empty; provide records with a key field');
|
|
36
|
+
}
|
|
37
|
+
if (primaryKeyFields.length === 1) {
|
|
38
|
+
const field = primaryKeyFields[0];
|
|
39
|
+
const value = payload[field];
|
|
40
|
+
if (value === undefined || value === null || String(value).trim() === '') {
|
|
41
|
+
throw new Error(`Missing primary key field '${field}' in record payload`);
|
|
42
|
+
}
|
|
43
|
+
return String(value);
|
|
44
|
+
}
|
|
45
|
+
const parts = [];
|
|
46
|
+
for (const field of primaryKeyFields) {
|
|
47
|
+
const value = payload[field];
|
|
48
|
+
if (value === undefined || value === null || String(value).trim() === '') {
|
|
49
|
+
throw new Error(`Missing composite primary key field '${field}' in record payload`);
|
|
50
|
+
}
|
|
51
|
+
parts.push(`${field}=${String(value)}`);
|
|
52
|
+
}
|
|
53
|
+
return parts.join(',');
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* @param {Object} datasource
|
|
58
|
+
* @param {Object} payload
|
|
59
|
+
* @returns {import('../api/types/records-bulk.types').ExternalBulkRecord}
|
|
60
|
+
*/
|
|
61
|
+
function mapPayloadToBulkRecord(datasource, payload) {
|
|
62
|
+
const primaryKey = Array.isArray(datasource.primaryKey) ? datasource.primaryKey : [];
|
|
63
|
+
const recordType =
|
|
64
|
+
typeof datasource.resourceType === 'string' && datasource.resourceType.trim()
|
|
65
|
+
? datasource.resourceType.trim()
|
|
66
|
+
: 'record';
|
|
67
|
+
const key = buildKeyFromPrimaryKey(primaryKey, payload);
|
|
68
|
+
let displayName = '';
|
|
69
|
+
const labelKey = datasource.labelKey;
|
|
70
|
+
if (Array.isArray(labelKey) && labelKey.length > 0) {
|
|
71
|
+
displayName = labelKey
|
|
72
|
+
.map(field => payload[field])
|
|
73
|
+
.filter(v => v !== undefined && v !== null && String(v).trim() !== '')
|
|
74
|
+
.map(v => String(v))
|
|
75
|
+
.join(' ');
|
|
76
|
+
} else if (typeof labelKey === 'string' && labelKey.trim() && payload[labelKey] !== undefined && payload[labelKey] !== null) {
|
|
77
|
+
displayName = String(payload[labelKey]);
|
|
78
|
+
}
|
|
79
|
+
const metadata = { ...payload };
|
|
80
|
+
if (metadata.externalId === undefined) {
|
|
81
|
+
metadata.externalId = key;
|
|
82
|
+
}
|
|
83
|
+
return {
|
|
84
|
+
key,
|
|
85
|
+
displayName: displayName || '',
|
|
86
|
+
recordType,
|
|
87
|
+
metadata
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* @param {Object[]} rawRecords
|
|
93
|
+
* @param {Object} datasource
|
|
94
|
+
* @returns {import('../api/types/records-bulk.types').ExternalBulkRecord[]}
|
|
95
|
+
*/
|
|
96
|
+
function normalizeRecordsForBulk(rawRecords, datasource) {
|
|
97
|
+
if (!Array.isArray(rawRecords) || rawRecords.length === 0) {
|
|
98
|
+
throw new Error('Record file is empty after parse');
|
|
99
|
+
}
|
|
100
|
+
return rawRecords.map((row, index) => {
|
|
101
|
+
try {
|
|
102
|
+
if (isCanonicalBulkRecord(row)) {
|
|
103
|
+
const meta = { ...row.metadata };
|
|
104
|
+
if (meta.externalId === undefined) {
|
|
105
|
+
meta.externalId = row.key;
|
|
106
|
+
}
|
|
107
|
+
return {
|
|
108
|
+
key: String(row.key).trim(),
|
|
109
|
+
displayName:
|
|
110
|
+
row.displayName !== undefined && row.displayName !== null
|
|
111
|
+
? String(row.displayName)
|
|
112
|
+
: '',
|
|
113
|
+
recordType: String(row.recordType).trim(),
|
|
114
|
+
metadata: meta
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
return mapPayloadToBulkRecord(datasource, row);
|
|
118
|
+
} catch (err) {
|
|
119
|
+
throw new Error(`Record ${index + 1}: ${err.message}`);
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
module.exports = {
|
|
125
|
+
isCanonicalBulkRecord,
|
|
126
|
+
buildKeyFromPrimaryKey,
|
|
127
|
+
mapPayloadToBulkRecord,
|
|
128
|
+
normalizeRecordsForBulk
|
|
129
|
+
};
|
|
@@ -5,49 +5,82 @@
|
|
|
5
5
|
* @version 2.0.0
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
const path = require('path');
|
|
9
8
|
const fs = require('fs');
|
|
10
9
|
const {
|
|
11
10
|
getIntegrationPath,
|
|
12
11
|
listIntegrationAppNames,
|
|
13
12
|
resolveIntegrationAppKeyFromCwd
|
|
14
13
|
} = require('../utils/paths');
|
|
15
|
-
const {
|
|
16
|
-
|
|
14
|
+
const {
|
|
15
|
+
appHasDatasourceIdentifier,
|
|
16
|
+
pickIntegrationAppForDatasourceIdentifier
|
|
17
|
+
} = require('./datasource-identifier');
|
|
18
|
+
|
|
19
|
+
/** @deprecated Use appHasDatasourceIdentifier — kept for tests and backward compatibility */
|
|
20
|
+
function appHasDatasourceKey(appKey, datasourceKey) {
|
|
21
|
+
return appHasDatasourceIdentifier(appKey, datasourceKey);
|
|
22
|
+
}
|
|
17
23
|
|
|
18
24
|
/**
|
|
19
|
-
*
|
|
20
|
-
* @param {string}
|
|
21
|
-
* @
|
|
22
|
-
* @returns {boolean} True if this app has a datasource with that key
|
|
25
|
+
* @param {string} key
|
|
26
|
+
* @param {string} [explicitApp]
|
|
27
|
+
* @returns {{ appKey: string }|null}
|
|
23
28
|
*/
|
|
24
|
-
function
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
function resolveFromExplicitApp(key, explicitApp) {
|
|
30
|
+
if (!explicitApp || typeof explicitApp !== 'string' || !explicitApp.trim()) {
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
const appKey = explicitApp.trim();
|
|
34
|
+
if (fs.existsSync(getIntegrationPath(appKey))) {
|
|
35
|
+
return { appKey };
|
|
36
|
+
}
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* @param {string} key
|
|
42
|
+
* @param {string[]} appNames
|
|
43
|
+
* @returns {{ appKey: string }|null}
|
|
44
|
+
* @throws {Error}
|
|
45
|
+
*/
|
|
46
|
+
function resolveFromAppScan(key, appNames) {
|
|
47
|
+
const matches = appNames.filter(appName => appHasDatasourceIdentifier(appName, key));
|
|
48
|
+
if (matches.length === 1) {
|
|
49
|
+
return { appKey: matches[0] };
|
|
50
|
+
}
|
|
51
|
+
if (matches.length > 1) {
|
|
52
|
+
throw new Error(
|
|
53
|
+
`More than one app has this datasource; add --app <app>. Apps: ${matches.join(', ')}`
|
|
54
|
+
);
|
|
32
55
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
// skip unreadable or invalid files
|
|
46
|
-
}
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* @param {string} key
|
|
61
|
+
* @param {string[]} appNames
|
|
62
|
+
* @returns {{ appKey: string }|null}
|
|
63
|
+
*/
|
|
64
|
+
function resolveFromPrefixOrSegments(key, appNames) {
|
|
65
|
+
const prefixApp = pickIntegrationAppForDatasourceIdentifier(key, appNames);
|
|
66
|
+
if (prefixApp && appHasDatasourceIdentifier(prefixApp, key)) {
|
|
67
|
+
return { appKey: prefixApp };
|
|
47
68
|
}
|
|
48
|
-
|
|
69
|
+
const segments = key.split('-');
|
|
70
|
+
if (segments.length < 2) {
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
const candidate = segments.slice(0, -1).join('-');
|
|
74
|
+
const candidatePath = getIntegrationPath(candidate);
|
|
75
|
+
if (fs.existsSync(candidatePath) && appHasDatasourceIdentifier(candidate, key)) {
|
|
76
|
+
return { appKey: candidate };
|
|
77
|
+
}
|
|
78
|
+
return null;
|
|
49
79
|
}
|
|
50
80
|
|
|
81
|
+
const RESOLVE_APP_HINT =
|
|
82
|
+
'Could not determine app context. Use --app <app>, a datasource key, a datasource JSON filename (e.g. myapp-datasource-contacts), or run from integration/<systemKey>/ directory.';
|
|
83
|
+
|
|
51
84
|
/**
|
|
52
85
|
* Resolve app key for a datasource: explicit --app, cwd, scan by key, or parse key convention.
|
|
53
86
|
* @async
|
|
@@ -56,7 +89,6 @@ function appHasDatasourceKey(appKey, datasourceKey) {
|
|
|
56
89
|
* @returns {Promise<{appKey: string}>} Resolved app key
|
|
57
90
|
* @throws {Error} When app cannot be determined or multiple apps match
|
|
58
91
|
*/
|
|
59
|
-
/* eslint-disable-next-line max-statements -- Resolution order: explicit, cwd, scan, parse */
|
|
60
92
|
async function resolveAppKeyForDatasource(datasourceKey, explicitApp) {
|
|
61
93
|
if (!datasourceKey || typeof datasourceKey !== 'string') {
|
|
62
94
|
throw new Error('Datasource key is required');
|
|
@@ -66,41 +98,28 @@ async function resolveAppKeyForDatasource(datasourceKey, explicitApp) {
|
|
|
66
98
|
throw new Error('Datasource key cannot be empty');
|
|
67
99
|
}
|
|
68
100
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
return { appKey: explicitApp.trim() };
|
|
73
|
-
}
|
|
101
|
+
const explicit = resolveFromExplicitApp(key, explicitApp);
|
|
102
|
+
if (explicit) {
|
|
103
|
+
return explicit;
|
|
74
104
|
}
|
|
75
105
|
|
|
76
106
|
const fromCwd = resolveIntegrationAppKeyFromCwd();
|
|
77
|
-
if (fromCwd &&
|
|
107
|
+
if (fromCwd && appHasDatasourceIdentifier(fromCwd, key)) {
|
|
78
108
|
return { appKey: fromCwd };
|
|
79
109
|
}
|
|
80
110
|
|
|
81
111
|
const appNames = listIntegrationAppNames();
|
|
82
|
-
const
|
|
83
|
-
if (
|
|
84
|
-
return
|
|
85
|
-
}
|
|
86
|
-
if (matches.length > 1) {
|
|
87
|
-
throw new Error(
|
|
88
|
-
`More than one app has this datasource; add --app <app>. Apps: ${matches.join(', ')}`
|
|
89
|
-
);
|
|
112
|
+
const scanned = resolveFromAppScan(key, appNames);
|
|
113
|
+
if (scanned) {
|
|
114
|
+
return scanned;
|
|
90
115
|
}
|
|
91
116
|
|
|
92
|
-
const
|
|
93
|
-
if (
|
|
94
|
-
|
|
95
|
-
const candidatePath = getIntegrationPath(candidate);
|
|
96
|
-
if (fs.existsSync(candidatePath) && appHasDatasourceKey(candidate, key)) {
|
|
97
|
-
return { appKey: candidate };
|
|
98
|
-
}
|
|
117
|
+
const inferred = resolveFromPrefixOrSegments(key, appNames);
|
|
118
|
+
if (inferred) {
|
|
119
|
+
return inferred;
|
|
99
120
|
}
|
|
100
121
|
|
|
101
|
-
throw new Error(
|
|
102
|
-
'Could not determine app context. Use --app <app> or run from integration/<systemKey>/ directory.'
|
|
103
|
-
);
|
|
122
|
+
throw new Error(RESOLVE_APP_HINT);
|
|
104
123
|
}
|
|
105
124
|
|
|
106
125
|
module.exports = {
|
|
@@ -11,7 +11,7 @@ const chalk = require('chalk');
|
|
|
11
11
|
const logger = require('../utils/logger');
|
|
12
12
|
const { getIntegrationPath } = require('../utils/paths');
|
|
13
13
|
const { resolveAppKeyForDatasource } = require('./resolve-app');
|
|
14
|
-
const { infoLine } = require('../utils/cli-test-layout-chalk');
|
|
14
|
+
const { formatProgress, formatSuccessLine, infoLine } = require('../utils/cli-test-layout-chalk');
|
|
15
15
|
const { runUnifiedDatasourceValidation } = require('./unified-validation-run');
|
|
16
16
|
const { includeDebugForRequest } = require('../utils/validation-run-request');
|
|
17
17
|
const { e2eShapeFromEnvelope } = require('../utils/datasource-test-run-legacy-adapter');
|
|
@@ -41,7 +41,9 @@ function buildUnifiedE2eRunOptions(options, timeoutMs, pk) {
|
|
|
41
41
|
minRecordCount: options.minRecordCount,
|
|
42
42
|
capabilityKey: options.capabilityKey,
|
|
43
43
|
timeout: timeoutMs,
|
|
44
|
-
sync: options.sync === true
|
|
44
|
+
sync: options.sync === true,
|
|
45
|
+
authConfig: options.authConfig,
|
|
46
|
+
dataplaneUrl: options.dataplaneUrl || options.dataplane
|
|
45
47
|
};
|
|
46
48
|
}
|
|
47
49
|
|
|
@@ -51,6 +53,25 @@ function logE2eDatasourceBanner(datasourceKey, verbose) {
|
|
|
51
53
|
logger.log(infoLine(`🧪 Running E2E test for datasource: ${datasourceKey}`));
|
|
52
54
|
}
|
|
53
55
|
|
|
56
|
+
/**
|
|
57
|
+
* Compact per-datasource progress (default test-e2e / system rollup when not --verbose).
|
|
58
|
+
* @param {string} datasourceKey
|
|
59
|
+
* @param {boolean} verbose
|
|
60
|
+
*/
|
|
61
|
+
function logE2eDatasourceProgress(datasourceKey, verbose) {
|
|
62
|
+
if (verbose) return;
|
|
63
|
+
logger.log(formatProgress(`Running E2E validation for ${datasourceKey}…`));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* @param {string} datasourceKey
|
|
68
|
+
* @param {boolean} verbose
|
|
69
|
+
*/
|
|
70
|
+
function logE2eDatasourceValidated(datasourceKey, verbose) {
|
|
71
|
+
if (verbose) return;
|
|
72
|
+
logger.log(formatSuccessLine(`${datasourceKey} validated`));
|
|
73
|
+
}
|
|
74
|
+
|
|
54
75
|
/**
|
|
55
76
|
* Resolve primaryKeyValue for request body: string as-is, or read and parse JSON from @path
|
|
56
77
|
* @param {string} [value]
|
|
@@ -174,6 +195,7 @@ async function runDatasourceTestE2E(datasourceKey, options = {}) {
|
|
|
174
195
|
const cliWallStartedAt = Date.now();
|
|
175
196
|
const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
|
|
176
197
|
logE2eDatasourceBanner(datasourceKey, options.verbose);
|
|
198
|
+
logE2eDatasourceProgress(datasourceKey, options.verbose);
|
|
177
199
|
const pk = await resolvePrimaryKeyValue(options.primaryKeyValue);
|
|
178
200
|
const timeoutMs = resolveE2ePollTimeoutMs(options);
|
|
179
201
|
const requestMeta = buildE2eRequestMeta(datasourceKey, options, pk);
|
|
@@ -184,6 +206,8 @@ async function runDatasourceTestE2E(datasourceKey, options = {}) {
|
|
|
184
206
|
|
|
185
207
|
await throwIfUnifiedE2EBlocked(unifiedResult, appKey, options, requestMeta);
|
|
186
208
|
|
|
209
|
+
logE2eDatasourceValidated(datasourceKey, options.verbose);
|
|
210
|
+
|
|
187
211
|
const display = e2eShapeFromEnvelope(unifiedResult.envelope);
|
|
188
212
|
Object.assign(display, { datasourceTestRun: unifiedResult.envelope });
|
|
189
213
|
|
|
@@ -4,40 +4,73 @@
|
|
|
4
4
|
* @version 2.0.0
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
const path = require('path');
|
|
8
7
|
const { getIntegrationPath } = require('../utils/paths');
|
|
9
8
|
const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
|
|
10
9
|
const { loadConfigFile } = require('../utils/config-format');
|
|
10
|
+
const {
|
|
11
|
+
datasourceIdentifierMatchesFile,
|
|
12
|
+
resolveDatasourceFilePath
|
|
13
|
+
} = require('./datasource-identifier');
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* @param {string} appPath
|
|
17
|
+
* @param {string} schemaBasePath
|
|
18
|
+
* @param {string[]} datasourceFiles
|
|
19
|
+
* @param {string} identifier
|
|
20
|
+
* @returns {string|null}
|
|
21
|
+
*/
|
|
22
|
+
function findDatasourceFileByIdentifier(appPath, schemaBasePath, datasourceFiles, identifier) {
|
|
23
|
+
const fsSync = require('fs');
|
|
24
|
+
for (const fileRef of datasourceFiles) {
|
|
25
|
+
if (!fileRef || typeof fileRef !== 'string') {
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
const fullPath = resolveDatasourceFilePath(appPath, schemaBasePath, fileRef);
|
|
29
|
+
if (!fsSync.existsSync(fullPath)) {
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
const parsed = loadConfigFile(fullPath);
|
|
34
|
+
if (datasourceIdentifierMatchesFile(identifier, fileRef, parsed)) {
|
|
35
|
+
return fileRef;
|
|
36
|
+
}
|
|
37
|
+
} catch {
|
|
38
|
+
// skip unreadable or invalid files
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
11
43
|
|
|
12
44
|
/**
|
|
13
45
|
* @param {string} appKey
|
|
14
46
|
* @param {string} datasourceKey
|
|
15
47
|
* @param {Function} findDatasourceFileByKey - From integration-context
|
|
16
|
-
* @returns {{ datasource: Object, datasourcePath: string }}
|
|
48
|
+
* @returns {{ datasource: Object, datasourcePath: string, datasourceKey: string }}
|
|
17
49
|
*/
|
|
18
50
|
function loadDatasourceForApp(appKey, datasourceKey, findDatasourceFileByKey) {
|
|
19
51
|
const appPath = getIntegrationPath(appKey);
|
|
20
52
|
const config = loadConfigFile(resolveApplicationConfigPath(appPath));
|
|
21
53
|
const schemaBasePath = config.externalIntegration?.schemaBasePath || './';
|
|
22
54
|
const datasourceFiles = config.externalIntegration?.dataSources || [];
|
|
23
|
-
let datasourceFile =
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
55
|
+
let datasourceFile = findDatasourceFileByIdentifier(
|
|
56
|
+
appPath,
|
|
57
|
+
schemaBasePath,
|
|
58
|
+
datasourceFiles,
|
|
59
|
+
datasourceKey
|
|
60
|
+
);
|
|
27
61
|
if (!datasourceFile) {
|
|
28
62
|
datasourceFile = findDatasourceFileByKey(appPath, schemaBasePath, datasourceFiles, datasourceKey);
|
|
29
63
|
}
|
|
30
64
|
if (!datasourceFile) {
|
|
31
65
|
throw new Error(`Datasource '${datasourceKey}' not found in application config`);
|
|
32
66
|
}
|
|
33
|
-
const datasourcePath =
|
|
34
|
-
? path.join(schemaBasePath, datasourceFile)
|
|
35
|
-
: path.join(appPath, schemaBasePath, datasourceFile);
|
|
67
|
+
const datasourcePath = resolveDatasourceFilePath(appPath, schemaBasePath, datasourceFile);
|
|
36
68
|
const datasource = loadConfigFile(datasourcePath);
|
|
37
|
-
|
|
38
|
-
|
|
69
|
+
const canonicalKey = datasource.key;
|
|
70
|
+
if (!canonicalKey || typeof canonicalKey !== 'string') {
|
|
71
|
+
throw new Error(`Datasource file is missing a valid key: ${datasourcePath}`);
|
|
39
72
|
}
|
|
40
|
-
return { datasource, datasourcePath };
|
|
73
|
+
return { datasource, datasourcePath, datasourceKey: canonicalKey };
|
|
41
74
|
}
|
|
42
75
|
|
|
43
76
|
module.exports = { loadDatasourceForApp };
|
|
@@ -71,6 +71,40 @@ function loadDatasourceForAppExport(appKey, datasourceKey) {
|
|
|
71
71
|
return loadDatasourceForApp(appKey, datasourceKey, findDatasourceFileByKey);
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
+
/**
|
|
75
|
+
* @async
|
|
76
|
+
* @param {string} appKey
|
|
77
|
+
* @param {Object} options
|
|
78
|
+
* @returns {Promise<{ authConfig: Object, dataplaneUrl: string }>}
|
|
79
|
+
*/
|
|
80
|
+
async function resolveValidationAuth(appKey, options) {
|
|
81
|
+
if (options.authConfig && (options.dataplaneUrl || options.dataplane)) {
|
|
82
|
+
return {
|
|
83
|
+
authConfig: options.authConfig,
|
|
84
|
+
dataplaneUrl: options.dataplaneUrl || options.dataplane
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
const configObj = await getConfig();
|
|
88
|
+
const auth = await setupIntegrationTestAuth(appKey, options, configObj);
|
|
89
|
+
return { authConfig: auth.authConfig, dataplaneUrl: auth.dataplaneUrl };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* @async
|
|
94
|
+
* @param {string} datasourceKey
|
|
95
|
+
* @param {Object} options
|
|
96
|
+
* @returns {Promise<Object>}
|
|
97
|
+
*/
|
|
98
|
+
async function prepareValidationRunContext(datasourceKey, options) {
|
|
99
|
+
const runType = options.runType || 'test';
|
|
100
|
+
const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
|
|
101
|
+
const systemKey = await getSystemKeyFromAppKey(appKey);
|
|
102
|
+
const { datasource } = loadDatasourceForAppExport(appKey, datasourceKey);
|
|
103
|
+
const { authConfig, dataplaneUrl } = await resolveValidationAuth(appKey, options);
|
|
104
|
+
await maybeSyncDatasourceToDataplane({ options, authConfig, systemKey });
|
|
105
|
+
return { runType, systemKey, datasource, authConfig, dataplaneUrl };
|
|
106
|
+
}
|
|
107
|
+
|
|
74
108
|
/**
|
|
75
109
|
* Run unified validation for one persisted datasource.
|
|
76
110
|
* @async
|
|
@@ -83,41 +117,33 @@ async function runUnifiedDatasourceValidation(datasourceKey, options) {
|
|
|
83
117
|
if (!datasourceKey || typeof datasourceKey !== 'string') {
|
|
84
118
|
throw new Error('Datasource key is required');
|
|
85
119
|
}
|
|
86
|
-
const runType = options.runType || 'test';
|
|
87
|
-
const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
|
|
88
|
-
const systemKey = await getSystemKeyFromAppKey(appKey);
|
|
89
|
-
const { datasource } = loadDatasourceForAppExport(appKey, datasourceKey);
|
|
90
|
-
|
|
91
|
-
const configObj = await getConfig();
|
|
92
|
-
const { authConfig, dataplaneUrl } = await setupIntegrationTestAuth(appKey, options, configObj);
|
|
93
|
-
|
|
94
|
-
await maybeSyncDatasourceToDataplane({ options, authConfig, systemKey });
|
|
95
120
|
|
|
121
|
+
const ctx = await prepareValidationRunContext(datasourceKey, options);
|
|
96
122
|
const useAsync = options.noAsync ? false : options.async !== false;
|
|
97
123
|
const timeoutMs = parseInt(String(options.timeout || '30000'), 10) || 30000;
|
|
98
|
-
|
|
99
124
|
const body = await buildUnifiedValidationBody({
|
|
100
|
-
systemKey,
|
|
125
|
+
systemKey: ctx.systemKey,
|
|
101
126
|
datasourceKey,
|
|
102
|
-
runType,
|
|
103
|
-
datasource,
|
|
127
|
+
runType: ctx.runType,
|
|
128
|
+
datasource: ctx.datasource,
|
|
104
129
|
payloadPath: options.payload,
|
|
105
130
|
useAsync,
|
|
106
131
|
options
|
|
107
132
|
});
|
|
108
133
|
|
|
109
134
|
const result = await postValidationRunAndOptionalPoll({
|
|
110
|
-
dataplaneUrl,
|
|
111
|
-
authConfig,
|
|
135
|
+
dataplaneUrl: ctx.dataplaneUrl,
|
|
136
|
+
authConfig: ctx.authConfig,
|
|
112
137
|
body,
|
|
113
138
|
timeoutMs,
|
|
114
139
|
useAsync,
|
|
115
140
|
noAsync: options.noAsync === true || options.async === false,
|
|
116
|
-
verbosePoll: options.verbose === true
|
|
141
|
+
verbosePoll: options.verbose === true,
|
|
142
|
+
pollSubject: datasourceKey
|
|
117
143
|
});
|
|
118
144
|
|
|
119
145
|
if (result && result.envelope) {
|
|
120
|
-
annotateMissingAutoRbacPermissions(result.envelope, datasourceKey, datasource);
|
|
146
|
+
annotateMissingAutoRbacPermissions(result.envelope, datasourceKey, ctx.datasource);
|
|
121
147
|
}
|
|
122
148
|
return result;
|
|
123
149
|
}
|
|
@@ -11,6 +11,11 @@
|
|
|
11
11
|
const path = require('path');
|
|
12
12
|
const fs = require('fs');
|
|
13
13
|
const pathsUtil = require('../utils/paths');
|
|
14
|
+
const {
|
|
15
|
+
datasourceIdentifierMatchesFile,
|
|
16
|
+
pickIntegrationAppForDatasourceIdentifier,
|
|
17
|
+
collectDatasourceFileRefsMatchingIdentifier
|
|
18
|
+
} = require('./datasource-identifier');
|
|
14
19
|
const { loadExternalDataSourceSchema } = require('../utils/schema-loader');
|
|
15
20
|
const { formatValidationErrors } = require('../utils/error-formatter');
|
|
16
21
|
const { validateFieldReferences } = require('./field-reference-validator');
|
|
@@ -46,16 +51,13 @@ function tryResolveAsExistingFile(identifier) {
|
|
|
46
51
|
}
|
|
47
52
|
|
|
48
53
|
/**
|
|
49
|
-
* Pick integration app folder for a datasource
|
|
54
|
+
* Pick integration app folder for a datasource identifier (longest matching prefix wins).
|
|
50
55
|
* @param {string} datasourceKey
|
|
51
56
|
* @param {string[]} appNames
|
|
52
57
|
* @returns {string|null}
|
|
53
58
|
*/
|
|
54
59
|
function pickIntegrationAppForDatasourceKey(datasourceKey, appNames) {
|
|
55
|
-
|
|
56
|
-
if (matches.length === 0) return null;
|
|
57
|
-
matches.sort((a, b) => b.length - a.length);
|
|
58
|
-
return matches[0];
|
|
60
|
+
return pickIntegrationAppForDatasourceIdentifier(datasourceKey, appNames);
|
|
59
61
|
}
|
|
60
62
|
|
|
61
63
|
/**
|
|
@@ -94,7 +96,7 @@ function collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey) {
|
|
|
94
96
|
} catch {
|
|
95
97
|
continue;
|
|
96
98
|
}
|
|
97
|
-
if (
|
|
99
|
+
if (datasourceIdentifierMatchesFile(datasourceKey, name, parsed)) {
|
|
98
100
|
hits.push(full);
|
|
99
101
|
}
|
|
100
102
|
}
|
|
@@ -102,21 +104,12 @@ function collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey) {
|
|
|
102
104
|
}
|
|
103
105
|
|
|
104
106
|
/**
|
|
105
|
-
*
|
|
107
|
+
* @param {string} appName
|
|
106
108
|
* @param {string} datasourceKey
|
|
107
|
-
* @returns {string}
|
|
109
|
+
* @returns {string}
|
|
108
110
|
* @throws {Error}
|
|
109
111
|
*/
|
|
110
|
-
function
|
|
111
|
-
const appNames = pathsUtil.listIntegrationAppNames();
|
|
112
|
-
const appName = pickIntegrationAppForDatasourceKey(datasourceKey, appNames);
|
|
113
|
-
if (!appName) {
|
|
114
|
-
throw new Error(
|
|
115
|
-
`No integration/<app>/ folder matches datasource key "${datasourceKey}". ` +
|
|
116
|
-
'Pass a path to the JSON file, or use a key whose prefix matches an app under integration/ ' +
|
|
117
|
-
'(e.g. app test-e2e-hubspot for key test-e2e-hubspot-users).'
|
|
118
|
-
);
|
|
119
|
-
}
|
|
112
|
+
function resolvePathFromIntegrationFolderScan(appName, datasourceKey) {
|
|
120
113
|
const dir = pathsUtil.getIntegrationPath(appName);
|
|
121
114
|
if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
|
|
122
115
|
throw new Error(`Integration folder not found: ${dir}`);
|
|
@@ -128,16 +121,40 @@ function resolvePathFromIntegrationDatasourceKey(datasourceKey) {
|
|
|
128
121
|
const hits = collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey);
|
|
129
122
|
if (hits.length === 0) {
|
|
130
123
|
throw new Error(
|
|
131
|
-
`No datasource JSON
|
|
124
|
+
`No datasource JSON matching "${datasourceKey}" under integration/${appName}/. ` +
|
|
132
125
|
`Checked: ${scanOrder.length ? scanOrder.join(', ') : '(no JSON candidates)'}`
|
|
133
126
|
);
|
|
134
127
|
}
|
|
135
128
|
if (hits.length > 1) {
|
|
136
|
-
throw new Error(`Ambiguous: multiple files
|
|
129
|
+
throw new Error(`Ambiguous: multiple files match "${datasourceKey}": ${hits.join(', ')}`);
|
|
137
130
|
}
|
|
138
131
|
return hits[0];
|
|
139
132
|
}
|
|
140
133
|
|
|
134
|
+
/**
|
|
135
|
+
* Find integration JSON whose parsed `key` equals datasourceKey.
|
|
136
|
+
* @param {string} datasourceKey
|
|
137
|
+
* @returns {string} Absolute file path
|
|
138
|
+
* @throws {Error}
|
|
139
|
+
*/
|
|
140
|
+
function resolvePathFromIntegrationDatasourceKey(datasourceKey) {
|
|
141
|
+
const appNames = pathsUtil.listIntegrationAppNames();
|
|
142
|
+
const appName = pickIntegrationAppForDatasourceKey(datasourceKey, appNames);
|
|
143
|
+
if (!appName) {
|
|
144
|
+
throw new Error(
|
|
145
|
+
`No integration/<app>/ folder matches datasource identifier "${datasourceKey}". ` +
|
|
146
|
+
'Pass a path to the JSON file, or use a datasource key or filename whose prefix matches an app under integration/ ' +
|
|
147
|
+
'(e.g. test-e2e-hubspot-companies or test-e2e-hubspot-datasource-companies for app test-e2e-hubspot).'
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
const manifestHits = collectDatasourceFileRefsMatchingIdentifier(appName, datasourceKey);
|
|
151
|
+
if (manifestHits.length === 1) {
|
|
152
|
+
const dir = pathsUtil.getIntegrationPath(appName);
|
|
153
|
+
return path.join(dir, manifestHits[0]);
|
|
154
|
+
}
|
|
155
|
+
return resolvePathFromIntegrationFolderScan(appName, datasourceKey);
|
|
156
|
+
}
|
|
157
|
+
|
|
141
158
|
/**
|
|
142
159
|
* @param {string} identifier - File path or datasource `key`
|
|
143
160
|
* @returns {string} Absolute path to JSON file
|