@aifabrix/builder 2.45.0 → 2.45.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/.cursor/rules/docs-rules.mdc +3 -1
  2. package/integration/hubspot-test/test-artifacts/wizard-hubspot-credential-real.yaml +1 -1
  3. package/integration/hubspot-test/test-artifacts/wizard-hubspot-env-vars.yaml +1 -1
  4. package/integration/hubspot-test/wizard-hubspot-e2e.yaml +1 -1
  5. package/jest.projects.js +44 -1
  6. package/lib/api/agent-metadata-validation.api.js +93 -0
  7. package/lib/api/audit-api-helpers.js +68 -0
  8. package/lib/api/audit.api.js +119 -0
  9. package/lib/api/auth-cache.api.js +44 -0
  10. package/lib/api/certificates.api.js +4 -4
  11. package/lib/api/credential.api.js +2 -2
  12. package/lib/api/dataplane-health.api.js +137 -0
  13. package/lib/api/dataplane-sync.api.js +27 -0
  14. package/lib/api/datasources-core.api.js +21 -18
  15. package/lib/api/datasources-extended.api.js +22 -22
  16. package/lib/api/dimensions.api.js +3 -0
  17. package/lib/api/external-systems.api.js +13 -13
  18. package/lib/api/governance-scenarios.api.js +39 -0
  19. package/lib/api/groups.api.js +79 -0
  20. package/lib/api/index.js +45 -2
  21. package/lib/api/pipeline.api.js +4 -4
  22. package/lib/api/protection.api.js +157 -0
  23. package/lib/api/records-bulk.api.js +30 -0
  24. package/lib/api/records-search-parse.js +49 -0
  25. package/lib/api/records-search.api.js +29 -0
  26. package/lib/api/types/agent-metadata-validation.types.js +67 -0
  27. package/lib/api/types/audit.types.js +72 -0
  28. package/lib/api/types/dataplane-health.types.js +27 -0
  29. package/lib/api/types/dimensions.types.js +2 -0
  30. package/lib/api/types/governance-scenarios.types.js +39 -0
  31. package/lib/api/types/protection.types.js +69 -0
  32. package/lib/api/types/records-bulk.types.js +35 -0
  33. package/lib/api/types/records-search.types.js +28 -0
  34. package/lib/api/user-groups.api.js +50 -0
  35. package/lib/api/users.api.js +95 -0
  36. package/lib/api/validation-run.api.js +3 -3
  37. package/lib/api/validation-runner.js +3 -3
  38. package/lib/api/wizard-platform.api.js +3 -3
  39. package/lib/api/wizard.api.js +21 -21
  40. package/lib/app/deploy.js +34 -19
  41. package/lib/cli/index.js +4 -2
  42. package/lib/cli/infra-guided.js +22 -3
  43. package/lib/cli/setup-app.help.js +50 -2
  44. package/lib/cli/setup-app.js +33 -24
  45. package/lib/cli/setup-app.test-commands.js +80 -21
  46. package/lib/cli/setup-auth.js +30 -3
  47. package/lib/cli/setup-external-system.js +37 -3
  48. package/lib/cli/setup-utility-help-after.js +45 -0
  49. package/lib/cli/setup-utility-resolve.js +2 -2
  50. package/lib/cli/setup-utility.js +99 -81
  51. package/lib/commands/auth-status-dataplane-version.js +231 -0
  52. package/lib/commands/auth-status-display.js +94 -0
  53. package/lib/commands/auth-status.js +70 -6
  54. package/lib/commands/credential-push.js +10 -5
  55. package/lib/commands/datasource-load-export-cli.js +178 -0
  56. package/lib/commands/datasource-test-trust-cli.js +114 -0
  57. package/lib/commands/datasource-unified-test-cli.js +25 -17
  58. package/lib/commands/datasource-unified-test-cli.options.js +18 -3
  59. package/lib/commands/datasource-unified-test-e2e-cli-helpers.js +5 -5
  60. package/lib/commands/datasource-verify-audit-cli.js +96 -0
  61. package/lib/commands/datasource.js +68 -1
  62. package/lib/commands/dimension-value.js +43 -13
  63. package/lib/commands/dimension.js +38 -7
  64. package/lib/commands/identity-apply-sync.js +131 -0
  65. package/lib/commands/identity-group-cli.js +146 -0
  66. package/lib/commands/identity-membership-role-cache.js +175 -0
  67. package/lib/commands/identity-shared.js +94 -0
  68. package/lib/commands/identity-user-cli.js +158 -0
  69. package/lib/commands/identity.js +43 -0
  70. package/lib/commands/login.js +46 -16
  71. package/lib/commands/protection-cli-leaves.js +286 -0
  72. package/lib/commands/protection.js +50 -0
  73. package/lib/commands/repair.js +13 -3
  74. package/lib/commands/test-e2e-external.js +40 -9
  75. package/lib/commands/test-governance-cli-options.js +47 -0
  76. package/lib/commands/test-governance-command-action.js +55 -0
  77. package/lib/commands/test-governance-external.js +88 -0
  78. package/lib/commands/test-trust-cli-options.js +32 -0
  79. package/lib/commands/test-trust-command-action.js +118 -0
  80. package/lib/commands/test-trust-external.js +127 -0
  81. package/lib/commands/upload.js +34 -13
  82. package/lib/commands/wizard-core.js +3 -0
  83. package/lib/core/config-device-dataplane.js +185 -0
  84. package/lib/datasource/agent-trust-debug-log.js +45 -0
  85. package/lib/datasource/agent-trust-fetch.js +115 -0
  86. package/lib/datasource/agent-trust-map.js +101 -0
  87. package/lib/datasource/agent-trust-mock.js +44 -0
  88. package/lib/datasource/agent-trust-run.js +93 -0
  89. package/lib/datasource/agent-trust-timeout.js +25 -0
  90. package/lib/datasource/audit-evidence-display.js +80 -0
  91. package/lib/datasource/audit-evidence-extract-debug.js +103 -0
  92. package/lib/datasource/audit-evidence-extract.js +177 -0
  93. package/lib/datasource/audit-evidence-matrix-rows.js +158 -0
  94. package/lib/datasource/audit-evidence-operation-aliases.js +63 -0
  95. package/lib/datasource/audit-evidence-poll.js +109 -0
  96. package/lib/datasource/audit-evidence-run.js +177 -0
  97. package/lib/datasource/audit-evidence-verify-helpers.js +167 -0
  98. package/lib/datasource/audit-evidence-verify-matrix.js +177 -0
  99. package/lib/datasource/audit-evidence-verify.js +108 -0
  100. package/lib/datasource/bulk-loader-run-batches.js +88 -0
  101. package/lib/datasource/bulk-loader-service-retry.js +52 -0
  102. package/lib/datasource/bulk-loader-service.js +127 -0
  103. package/lib/datasource/capability/run-capability-dimension.js +2 -1
  104. package/lib/datasource/datasource-exporter-resolve.js +82 -0
  105. package/lib/datasource/datasource-exporter-service.js +144 -0
  106. package/lib/datasource/datasource-identifier.js +148 -0
  107. package/lib/datasource/datasource-load-export-context.js +92 -0
  108. package/lib/datasource/integration-context.js +6 -4
  109. package/lib/datasource/local-data-paths.js +145 -0
  110. package/lib/datasource/log-cleaner.js +241 -0
  111. package/lib/datasource/log-viewer.js +111 -6
  112. package/lib/datasource/record-file-parser.js +85 -0
  113. package/lib/datasource/record-mapper.js +129 -0
  114. package/lib/datasource/resolve-app.js +74 -55
  115. package/lib/datasource/test-e2e.js +26 -2
  116. package/lib/datasource/unified-validation-run-resolve.js +45 -12
  117. package/lib/datasource/unified-validation-run.js +43 -17
  118. package/lib/datasource/validate.js +37 -20
  119. package/lib/external-system/test-system-level.js +25 -4
  120. package/lib/external-system/test.js +3 -1
  121. package/lib/governance/governance-pack-loader.js +70 -0
  122. package/lib/governance/governance-run-display.js +119 -0
  123. package/lib/identity/identity-apply-core.js +108 -0
  124. package/lib/identity/identity-apply-phases.js +244 -0
  125. package/lib/identity/identity-apply-service.js +77 -0
  126. package/lib/identity/identity-csv-parser.js +114 -0
  127. package/lib/internal/fs-real-sync.js +29 -1
  128. package/lib/protection/auth-context.js +45 -0
  129. package/lib/protection/convert-batch.js +90 -0
  130. package/lib/protection/load.js +41 -0
  131. package/lib/protection/paths.js +158 -0
  132. package/lib/protection/preflight-datasource-ready.js +46 -0
  133. package/lib/protection/protection-create-probe-lines.js +142 -0
  134. package/lib/protection/protection-create-probes.js +95 -0
  135. package/lib/protection/protection-create-scaffold.js +281 -0
  136. package/lib/protection/protection-display-helpers.js +445 -0
  137. package/lib/protection/protection-display.js +215 -0
  138. package/lib/protection/protection-preset-registry.js +97 -0
  139. package/lib/protection/report-exit.js +62 -0
  140. package/lib/protection/resolve.js +202 -0
  141. package/lib/protection/run-commands-validate.js +130 -0
  142. package/lib/protection/run-commands.js +216 -0
  143. package/lib/protection/run-protection-create-helpers.js +134 -0
  144. package/lib/protection/run-protection-create-probes-flow.js +43 -0
  145. package/lib/protection/run-protection-create.js +191 -0
  146. package/lib/protection/scope.js +22 -0
  147. package/lib/protection/sync-after-upload.js +64 -0
  148. package/lib/protection/upload-batch.js +127 -0
  149. package/lib/protection/validate-batch.js +113 -0
  150. package/lib/protection/validate-local.js +65 -0
  151. package/lib/resolvers/dimension-file.js +29 -4
  152. package/lib/schema/external-datasource.schema.json +47 -13
  153. package/lib/schema/flag-map-validation-run.json +2 -2
  154. package/lib/schema/protection.schema.json +374 -0
  155. package/lib/schema/type/credential-field-registry.json +636 -0
  156. package/lib/schema/wizard-config.schema.json +1 -1
  157. package/lib/utils/agent-trust-run-display.js +231 -0
  158. package/lib/utils/agent-trust-run-exit.js +59 -0
  159. package/lib/utils/agent-trust-wait-ui.js +94 -0
  160. package/lib/utils/cli-exit-codes.js +30 -0
  161. package/lib/utils/cli-sync-options.js +33 -0
  162. package/lib/utils/credential-push-ui.js +90 -0
  163. package/lib/utils/dataplane-cli-version-gate.js +202 -0
  164. package/lib/utils/dataplane-cli-version-help.js +74 -0
  165. package/lib/utils/datasource-export-display.js +137 -0
  166. package/lib/utils/datasource-load-display.js +174 -0
  167. package/lib/utils/datasource-load-export-ui.js +120 -0
  168. package/lib/utils/datasource-test-run-capability-scope.js +3 -3
  169. package/lib/utils/datasource-test-run-display.js +3 -3
  170. package/lib/utils/datasource-test-run-tty-log.js +5 -1
  171. package/lib/utils/help-builder.js +30 -24
  172. package/lib/utils/manifest-source-emit.js +22 -0
  173. package/lib/utils/semver-compare.js +127 -0
  174. package/lib/utils/upload-pipeline-ui.js +91 -0
  175. package/lib/utils/validation-poll-ui.js +14 -7
  176. package/lib/validation/wizard-config-validator.js +58 -19
  177. package/package.json +1 -1
  178. package/templates/applications/dataplane/env.template +13 -1
  179. package/templates/applications/dataplane/rbac.yaml +4 -0
  180. package/templates/external-system/README.md.hbs +56 -8
@@ -0,0 +1,129 @@
1
+ /**
2
+ * @fileoverview Map fixture payloads to canonical bulk sync records (plan 144)
3
+ * @author AI Fabrix Team
4
+ * @version 1.0.0
5
+ */
6
+
7
+ 'use strict';
8
+
9
+ /**
10
+ * @param {Object} obj
11
+ * @returns {boolean}
12
+ */
13
+ function isCanonicalBulkRecord(obj) {
14
+ return (
15
+ obj &&
16
+ typeof obj === 'object' &&
17
+ typeof obj.key === 'string' &&
18
+ obj.key.trim().length > 0 &&
19
+ typeof obj.recordType === 'string' &&
20
+ obj.recordType.trim().length > 0 &&
21
+ obj.metadata !== undefined &&
22
+ obj.metadata !== null &&
23
+ typeof obj.metadata === 'object' &&
24
+ !Array.isArray(obj.metadata)
25
+ );
26
+ }
27
+
28
+ /**
29
+ * @param {string[]} primaryKeyFields
30
+ * @param {Object} payload
31
+ * @returns {string}
32
+ */
33
+ function buildKeyFromPrimaryKey(primaryKeyFields, payload) {
34
+ if (!primaryKeyFields.length) {
35
+ throw new Error('Datasource primaryKey is empty; provide records with a key field');
36
+ }
37
+ if (primaryKeyFields.length === 1) {
38
+ const field = primaryKeyFields[0];
39
+ const value = payload[field];
40
+ if (value === undefined || value === null || String(value).trim() === '') {
41
+ throw new Error(`Missing primary key field '${field}' in record payload`);
42
+ }
43
+ return String(value);
44
+ }
45
+ const parts = [];
46
+ for (const field of primaryKeyFields) {
47
+ const value = payload[field];
48
+ if (value === undefined || value === null || String(value).trim() === '') {
49
+ throw new Error(`Missing composite primary key field '${field}' in record payload`);
50
+ }
51
+ parts.push(`${field}=${String(value)}`);
52
+ }
53
+ return parts.join(',');
54
+ }
55
+
56
+ /**
57
+ * @param {Object} datasource
58
+ * @param {Object} payload
59
+ * @returns {import('../api/types/records-bulk.types').ExternalBulkRecord}
60
+ */
61
+ function mapPayloadToBulkRecord(datasource, payload) {
62
+ const primaryKey = Array.isArray(datasource.primaryKey) ? datasource.primaryKey : [];
63
+ const recordType =
64
+ typeof datasource.resourceType === 'string' && datasource.resourceType.trim()
65
+ ? datasource.resourceType.trim()
66
+ : 'record';
67
+ const key = buildKeyFromPrimaryKey(primaryKey, payload);
68
+ let displayName = '';
69
+ const labelKey = datasource.labelKey;
70
+ if (Array.isArray(labelKey) && labelKey.length > 0) {
71
+ displayName = labelKey
72
+ .map(field => payload[field])
73
+ .filter(v => v !== undefined && v !== null && String(v).trim() !== '')
74
+ .map(v => String(v))
75
+ .join(' ');
76
+ } else if (typeof labelKey === 'string' && labelKey.trim() && payload[labelKey] !== undefined && payload[labelKey] !== null) {
77
+ displayName = String(payload[labelKey]);
78
+ }
79
+ const metadata = { ...payload };
80
+ if (metadata.externalId === undefined) {
81
+ metadata.externalId = key;
82
+ }
83
+ return {
84
+ key,
85
+ displayName: displayName || '',
86
+ recordType,
87
+ metadata
88
+ };
89
+ }
90
+
91
+ /**
92
+ * @param {Object[]} rawRecords
93
+ * @param {Object} datasource
94
+ * @returns {import('../api/types/records-bulk.types').ExternalBulkRecord[]}
95
+ */
96
+ function normalizeRecordsForBulk(rawRecords, datasource) {
97
+ if (!Array.isArray(rawRecords) || rawRecords.length === 0) {
98
+ throw new Error('Record file is empty after parse');
99
+ }
100
+ return rawRecords.map((row, index) => {
101
+ try {
102
+ if (isCanonicalBulkRecord(row)) {
103
+ const meta = { ...row.metadata };
104
+ if (meta.externalId === undefined) {
105
+ meta.externalId = row.key;
106
+ }
107
+ return {
108
+ key: String(row.key).trim(),
109
+ displayName:
110
+ row.displayName !== undefined && row.displayName !== null
111
+ ? String(row.displayName)
112
+ : '',
113
+ recordType: String(row.recordType).trim(),
114
+ metadata: meta
115
+ };
116
+ }
117
+ return mapPayloadToBulkRecord(datasource, row);
118
+ } catch (err) {
119
+ throw new Error(`Record ${index + 1}: ${err.message}`);
120
+ }
121
+ });
122
+ }
123
+
124
+ module.exports = {
125
+ isCanonicalBulkRecord,
126
+ buildKeyFromPrimaryKey,
127
+ mapPayloadToBulkRecord,
128
+ normalizeRecordsForBulk
129
+ };
@@ -5,49 +5,82 @@
5
5
  * @version 2.0.0
6
6
  */
7
7
 
8
- const path = require('path');
9
8
  const fs = require('fs');
10
9
  const {
11
10
  getIntegrationPath,
12
11
  listIntegrationAppNames,
13
12
  resolveIntegrationAppKeyFromCwd
14
13
  } = require('../utils/paths');
15
- const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
16
- const { loadConfigFile } = require('../utils/config-format');
14
+ const {
15
+ appHasDatasourceIdentifier,
16
+ pickIntegrationAppForDatasourceIdentifier
17
+ } = require('./datasource-identifier');
18
+
19
+ /** @deprecated Use appHasDatasourceIdentifier — kept for tests and backward compatibility */
20
+ function appHasDatasourceKey(appKey, datasourceKey) {
21
+ return appHasDatasourceIdentifier(appKey, datasourceKey);
22
+ }
17
23
 
18
24
  /**
19
- * For one app, check if any of its datasource files has the given key
20
- * @param {string} appKey - Integration folder name (system key context)
21
- * @param {string} datasourceKey - Datasource key to match
22
- * @returns {boolean} True if this app has a datasource with that key
25
+ * @param {string} key
26
+ * @param {string} [explicitApp]
27
+ * @returns {{ appKey: string }|null}
23
28
  */
24
- function appHasDatasourceKey(appKey, datasourceKey) {
25
- const appPath = getIntegrationPath(appKey);
26
- let config;
27
- try {
28
- const configPath = resolveApplicationConfigPath(appPath);
29
- config = loadConfigFile(configPath);
30
- } catch {
31
- return false;
29
+ function resolveFromExplicitApp(key, explicitApp) {
30
+ if (!explicitApp || typeof explicitApp !== 'string' || !explicitApp.trim()) {
31
+ return null;
32
+ }
33
+ const appKey = explicitApp.trim();
34
+ if (fs.existsSync(getIntegrationPath(appKey))) {
35
+ return { appKey };
36
+ }
37
+ return null;
38
+ }
39
+
40
+ /**
41
+ * @param {string} key
42
+ * @param {string[]} appNames
43
+ * @returns {{ appKey: string }|null}
44
+ * @throws {Error}
45
+ */
46
+ function resolveFromAppScan(key, appNames) {
47
+ const matches = appNames.filter(appName => appHasDatasourceIdentifier(appName, key));
48
+ if (matches.length === 1) {
49
+ return { appKey: matches[0] };
50
+ }
51
+ if (matches.length > 1) {
52
+ throw new Error(
53
+ `More than one app has this datasource; add --app <app>. Apps: ${matches.join(', ')}`
54
+ );
32
55
  }
33
- const schemaBasePath = config.externalIntegration?.schemaBasePath || './';
34
- const datasourceFiles = config.externalIntegration?.dataSources || [];
35
- for (const f of datasourceFiles) {
36
- if (!f || typeof f !== 'string') continue;
37
- const fullPath = path.isAbsolute(schemaBasePath)
38
- ? path.join(schemaBasePath, f)
39
- : path.join(appPath, schemaBasePath, f);
40
- if (!fs.existsSync(fullPath)) continue;
41
- try {
42
- const parsed = loadConfigFile(fullPath);
43
- if (parsed && parsed.key === datasourceKey) return true;
44
- } catch {
45
- // skip unreadable or invalid files
46
- }
56
+ return null;
57
+ }
58
+
59
+ /**
60
+ * @param {string} key
61
+ * @param {string[]} appNames
62
+ * @returns {{ appKey: string }|null}
63
+ */
64
+ function resolveFromPrefixOrSegments(key, appNames) {
65
+ const prefixApp = pickIntegrationAppForDatasourceIdentifier(key, appNames);
66
+ if (prefixApp && appHasDatasourceIdentifier(prefixApp, key)) {
67
+ return { appKey: prefixApp };
47
68
  }
48
- return false;
69
+ const segments = key.split('-');
70
+ if (segments.length < 2) {
71
+ return null;
72
+ }
73
+ const candidate = segments.slice(0, -1).join('-');
74
+ const candidatePath = getIntegrationPath(candidate);
75
+ if (fs.existsSync(candidatePath) && appHasDatasourceIdentifier(candidate, key)) {
76
+ return { appKey: candidate };
77
+ }
78
+ return null;
49
79
  }
50
80
 
81
+ const RESOLVE_APP_HINT =
82
+ 'Could not determine app context. Use --app <app>, a datasource key, a datasource JSON filename (e.g. myapp-datasource-contacts), or run from integration/<systemKey>/ directory.';
83
+
51
84
  /**
52
85
  * Resolve app key for a datasource: explicit --app, cwd, scan by key, or parse key convention.
53
86
  * @async
@@ -56,7 +89,6 @@ function appHasDatasourceKey(appKey, datasourceKey) {
56
89
  * @returns {Promise<{appKey: string}>} Resolved app key
57
90
  * @throws {Error} When app cannot be determined or multiple apps match
58
91
  */
59
- /* eslint-disable-next-line max-statements -- Resolution order: explicit, cwd, scan, parse */
60
92
  async function resolveAppKeyForDatasource(datasourceKey, explicitApp) {
61
93
  if (!datasourceKey || typeof datasourceKey !== 'string') {
62
94
  throw new Error('Datasource key is required');
@@ -66,41 +98,28 @@ async function resolveAppKeyForDatasource(datasourceKey, explicitApp) {
66
98
  throw new Error('Datasource key cannot be empty');
67
99
  }
68
100
 
69
- if (explicitApp && typeof explicitApp === 'string' && explicitApp.trim()) {
70
- const appPath = getIntegrationPath(explicitApp.trim());
71
- if (fs.existsSync(appPath)) {
72
- return { appKey: explicitApp.trim() };
73
- }
101
+ const explicit = resolveFromExplicitApp(key, explicitApp);
102
+ if (explicit) {
103
+ return explicit;
74
104
  }
75
105
 
76
106
  const fromCwd = resolveIntegrationAppKeyFromCwd();
77
- if (fromCwd && appHasDatasourceKey(fromCwd, key)) {
107
+ if (fromCwd && appHasDatasourceIdentifier(fromCwd, key)) {
78
108
  return { appKey: fromCwd };
79
109
  }
80
110
 
81
111
  const appNames = listIntegrationAppNames();
82
- const matches = appNames.filter(appName => appHasDatasourceKey(appName, key));
83
- if (matches.length === 1) {
84
- return { appKey: matches[0] };
85
- }
86
- if (matches.length > 1) {
87
- throw new Error(
88
- `More than one app has this datasource; add --app <app>. Apps: ${matches.join(', ')}`
89
- );
112
+ const scanned = resolveFromAppScan(key, appNames);
113
+ if (scanned) {
114
+ return scanned;
90
115
  }
91
116
 
92
- const segments = key.split('-');
93
- if (segments.length >= 2) {
94
- const candidate = segments.slice(0, -1).join('-');
95
- const candidatePath = getIntegrationPath(candidate);
96
- if (fs.existsSync(candidatePath) && appHasDatasourceKey(candidate, key)) {
97
- return { appKey: candidate };
98
- }
117
+ const inferred = resolveFromPrefixOrSegments(key, appNames);
118
+ if (inferred) {
119
+ return inferred;
99
120
  }
100
121
 
101
- throw new Error(
102
- 'Could not determine app context. Use --app <app> or run from integration/<systemKey>/ directory.'
103
- );
122
+ throw new Error(RESOLVE_APP_HINT);
104
123
  }
105
124
 
106
125
  module.exports = {
@@ -11,7 +11,7 @@ const chalk = require('chalk');
11
11
  const logger = require('../utils/logger');
12
12
  const { getIntegrationPath } = require('../utils/paths');
13
13
  const { resolveAppKeyForDatasource } = require('./resolve-app');
14
- const { infoLine } = require('../utils/cli-test-layout-chalk');
14
+ const { formatProgress, formatSuccessLine, infoLine } = require('../utils/cli-test-layout-chalk');
15
15
  const { runUnifiedDatasourceValidation } = require('./unified-validation-run');
16
16
  const { includeDebugForRequest } = require('../utils/validation-run-request');
17
17
  const { e2eShapeFromEnvelope } = require('../utils/datasource-test-run-legacy-adapter');
@@ -41,7 +41,9 @@ function buildUnifiedE2eRunOptions(options, timeoutMs, pk) {
41
41
  minRecordCount: options.minRecordCount,
42
42
  capabilityKey: options.capabilityKey,
43
43
  timeout: timeoutMs,
44
- sync: options.sync === true
44
+ sync: options.sync === true,
45
+ authConfig: options.authConfig,
46
+ dataplaneUrl: options.dataplaneUrl || options.dataplane
45
47
  };
46
48
  }
47
49
 
@@ -51,6 +53,25 @@ function logE2eDatasourceBanner(datasourceKey, verbose) {
51
53
  logger.log(infoLine(`🧪 Running E2E test for datasource: ${datasourceKey}`));
52
54
  }
53
55
 
56
+ /**
57
+ * Compact per-datasource progress (default test-e2e / system rollup when not --verbose).
58
+ * @param {string} datasourceKey
59
+ * @param {boolean} verbose
60
+ */
61
+ function logE2eDatasourceProgress(datasourceKey, verbose) {
62
+ if (verbose) return;
63
+ logger.log(formatProgress(`Running E2E validation for ${datasourceKey}…`));
64
+ }
65
+
66
+ /**
67
+ * @param {string} datasourceKey
68
+ * @param {boolean} verbose
69
+ */
70
+ function logE2eDatasourceValidated(datasourceKey, verbose) {
71
+ if (verbose) return;
72
+ logger.log(formatSuccessLine(`${datasourceKey} validated`));
73
+ }
74
+
54
75
  /**
55
76
  * Resolve primaryKeyValue for request body: string as-is, or read and parse JSON from @path
56
77
  * @param {string} [value]
@@ -174,6 +195,7 @@ async function runDatasourceTestE2E(datasourceKey, options = {}) {
174
195
  const cliWallStartedAt = Date.now();
175
196
  const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
176
197
  logE2eDatasourceBanner(datasourceKey, options.verbose);
198
+ logE2eDatasourceProgress(datasourceKey, options.verbose);
177
199
  const pk = await resolvePrimaryKeyValue(options.primaryKeyValue);
178
200
  const timeoutMs = resolveE2ePollTimeoutMs(options);
179
201
  const requestMeta = buildE2eRequestMeta(datasourceKey, options, pk);
@@ -184,6 +206,8 @@ async function runDatasourceTestE2E(datasourceKey, options = {}) {
184
206
 
185
207
  await throwIfUnifiedE2EBlocked(unifiedResult, appKey, options, requestMeta);
186
208
 
209
+ logE2eDatasourceValidated(datasourceKey, options.verbose);
210
+
187
211
  const display = e2eShapeFromEnvelope(unifiedResult.envelope);
188
212
  Object.assign(display, { datasourceTestRun: unifiedResult.envelope });
189
213
 
@@ -4,40 +4,73 @@
4
4
  * @version 2.0.0
5
5
  */
6
6
 
7
- const path = require('path');
8
7
  const { getIntegrationPath } = require('../utils/paths');
9
8
  const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
10
9
  const { loadConfigFile } = require('../utils/config-format');
10
+ const {
11
+ datasourceIdentifierMatchesFile,
12
+ resolveDatasourceFilePath
13
+ } = require('./datasource-identifier');
14
+
15
+ /**
16
+ * @param {string} appPath
17
+ * @param {string} schemaBasePath
18
+ * @param {string[]} datasourceFiles
19
+ * @param {string} identifier
20
+ * @returns {string|null}
21
+ */
22
+ function findDatasourceFileByIdentifier(appPath, schemaBasePath, datasourceFiles, identifier) {
23
+ const fsSync = require('fs');
24
+ for (const fileRef of datasourceFiles) {
25
+ if (!fileRef || typeof fileRef !== 'string') {
26
+ continue;
27
+ }
28
+ const fullPath = resolveDatasourceFilePath(appPath, schemaBasePath, fileRef);
29
+ if (!fsSync.existsSync(fullPath)) {
30
+ continue;
31
+ }
32
+ try {
33
+ const parsed = loadConfigFile(fullPath);
34
+ if (datasourceIdentifierMatchesFile(identifier, fileRef, parsed)) {
35
+ return fileRef;
36
+ }
37
+ } catch {
38
+ // skip unreadable or invalid files
39
+ }
40
+ }
41
+ return null;
42
+ }
11
43
 
12
44
  /**
13
45
  * @param {string} appKey
14
46
  * @param {string} datasourceKey
15
47
  * @param {Function} findDatasourceFileByKey - From integration-context
16
- * @returns {{ datasource: Object, datasourcePath: string }}
48
+ * @returns {{ datasource: Object, datasourcePath: string, datasourceKey: string }}
17
49
  */
18
50
  function loadDatasourceForApp(appKey, datasourceKey, findDatasourceFileByKey) {
19
51
  const appPath = getIntegrationPath(appKey);
20
52
  const config = loadConfigFile(resolveApplicationConfigPath(appPath));
21
53
  const schemaBasePath = config.externalIntegration?.schemaBasePath || './';
22
54
  const datasourceFiles = config.externalIntegration?.dataSources || [];
23
- let datasourceFile = datasourceFiles.find(f => {
24
- const base = path.basename(f, path.extname(f));
25
- return base === datasourceKey || base.includes(datasourceKey);
26
- });
55
+ let datasourceFile = findDatasourceFileByIdentifier(
56
+ appPath,
57
+ schemaBasePath,
58
+ datasourceFiles,
59
+ datasourceKey
60
+ );
27
61
  if (!datasourceFile) {
28
62
  datasourceFile = findDatasourceFileByKey(appPath, schemaBasePath, datasourceFiles, datasourceKey);
29
63
  }
30
64
  if (!datasourceFile) {
31
65
  throw new Error(`Datasource '${datasourceKey}' not found in application config`);
32
66
  }
33
- const datasourcePath = path.isAbsolute(schemaBasePath)
34
- ? path.join(schemaBasePath, datasourceFile)
35
- : path.join(appPath, schemaBasePath, datasourceFile);
67
+ const datasourcePath = resolveDatasourceFilePath(appPath, schemaBasePath, datasourceFile);
36
68
  const datasource = loadConfigFile(datasourcePath);
37
- if (datasource.key !== datasourceKey) {
38
- throw new Error(`Datasource key mismatch: file has '${datasource.key}', expected '${datasourceKey}'`);
69
+ const canonicalKey = datasource.key;
70
+ if (!canonicalKey || typeof canonicalKey !== 'string') {
71
+ throw new Error(`Datasource file is missing a valid key: ${datasourcePath}`);
39
72
  }
40
- return { datasource, datasourcePath };
73
+ return { datasource, datasourcePath, datasourceKey: canonicalKey };
41
74
  }
42
75
 
43
76
  module.exports = { loadDatasourceForApp };
@@ -71,6 +71,40 @@ function loadDatasourceForAppExport(appKey, datasourceKey) {
71
71
  return loadDatasourceForApp(appKey, datasourceKey, findDatasourceFileByKey);
72
72
  }
73
73
 
74
+ /**
75
+ * @async
76
+ * @param {string} appKey
77
+ * @param {Object} options
78
+ * @returns {Promise<{ authConfig: Object, dataplaneUrl: string }>}
79
+ */
80
+ async function resolveValidationAuth(appKey, options) {
81
+ if (options.authConfig && (options.dataplaneUrl || options.dataplane)) {
82
+ return {
83
+ authConfig: options.authConfig,
84
+ dataplaneUrl: options.dataplaneUrl || options.dataplane
85
+ };
86
+ }
87
+ const configObj = await getConfig();
88
+ const auth = await setupIntegrationTestAuth(appKey, options, configObj);
89
+ return { authConfig: auth.authConfig, dataplaneUrl: auth.dataplaneUrl };
90
+ }
91
+
92
+ /**
93
+ * @async
94
+ * @param {string} datasourceKey
95
+ * @param {Object} options
96
+ * @returns {Promise<Object>}
97
+ */
98
+ async function prepareValidationRunContext(datasourceKey, options) {
99
+ const runType = options.runType || 'test';
100
+ const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
101
+ const systemKey = await getSystemKeyFromAppKey(appKey);
102
+ const { datasource } = loadDatasourceForAppExport(appKey, datasourceKey);
103
+ const { authConfig, dataplaneUrl } = await resolveValidationAuth(appKey, options);
104
+ await maybeSyncDatasourceToDataplane({ options, authConfig, systemKey });
105
+ return { runType, systemKey, datasource, authConfig, dataplaneUrl };
106
+ }
107
+
74
108
  /**
75
109
  * Run unified validation for one persisted datasource.
76
110
  * @async
@@ -83,41 +117,33 @@ async function runUnifiedDatasourceValidation(datasourceKey, options) {
83
117
  if (!datasourceKey || typeof datasourceKey !== 'string') {
84
118
  throw new Error('Datasource key is required');
85
119
  }
86
- const runType = options.runType || 'test';
87
- const { appKey } = await resolveAppKeyForDatasource(datasourceKey, options.app);
88
- const systemKey = await getSystemKeyFromAppKey(appKey);
89
- const { datasource } = loadDatasourceForAppExport(appKey, datasourceKey);
90
-
91
- const configObj = await getConfig();
92
- const { authConfig, dataplaneUrl } = await setupIntegrationTestAuth(appKey, options, configObj);
93
-
94
- await maybeSyncDatasourceToDataplane({ options, authConfig, systemKey });
95
120
 
121
+ const ctx = await prepareValidationRunContext(datasourceKey, options);
96
122
  const useAsync = options.noAsync ? false : options.async !== false;
97
123
  const timeoutMs = parseInt(String(options.timeout || '30000'), 10) || 30000;
98
-
99
124
  const body = await buildUnifiedValidationBody({
100
- systemKey,
125
+ systemKey: ctx.systemKey,
101
126
  datasourceKey,
102
- runType,
103
- datasource,
127
+ runType: ctx.runType,
128
+ datasource: ctx.datasource,
104
129
  payloadPath: options.payload,
105
130
  useAsync,
106
131
  options
107
132
  });
108
133
 
109
134
  const result = await postValidationRunAndOptionalPoll({
110
- dataplaneUrl,
111
- authConfig,
135
+ dataplaneUrl: ctx.dataplaneUrl,
136
+ authConfig: ctx.authConfig,
112
137
  body,
113
138
  timeoutMs,
114
139
  useAsync,
115
140
  noAsync: options.noAsync === true || options.async === false,
116
- verbosePoll: options.verbose === true
141
+ verbosePoll: options.verbose === true,
142
+ pollSubject: datasourceKey
117
143
  });
118
144
 
119
145
  if (result && result.envelope) {
120
- annotateMissingAutoRbacPermissions(result.envelope, datasourceKey, datasource);
146
+ annotateMissingAutoRbacPermissions(result.envelope, datasourceKey, ctx.datasource);
121
147
  }
122
148
  return result;
123
149
  }
@@ -11,6 +11,11 @@
11
11
  const path = require('path');
12
12
  const fs = require('fs');
13
13
  const pathsUtil = require('../utils/paths');
14
+ const {
15
+ datasourceIdentifierMatchesFile,
16
+ pickIntegrationAppForDatasourceIdentifier,
17
+ collectDatasourceFileRefsMatchingIdentifier
18
+ } = require('./datasource-identifier');
14
19
  const { loadExternalDataSourceSchema } = require('../utils/schema-loader');
15
20
  const { formatValidationErrors } = require('../utils/error-formatter');
16
21
  const { validateFieldReferences } = require('./field-reference-validator');
@@ -46,16 +51,13 @@ function tryResolveAsExistingFile(identifier) {
46
51
  }
47
52
 
48
53
  /**
49
- * Pick integration app folder for a datasource `key` (longest matching prefix wins).
54
+ * Pick integration app folder for a datasource identifier (longest matching prefix wins).
50
55
  * @param {string} datasourceKey
51
56
  * @param {string[]} appNames
52
57
  * @returns {string|null}
53
58
  */
54
59
  function pickIntegrationAppForDatasourceKey(datasourceKey, appNames) {
55
- const matches = appNames.filter((app) => datasourceKey === app || datasourceKey.startsWith(`${app}-`));
56
- if (matches.length === 0) return null;
57
- matches.sort((a, b) => b.length - a.length);
58
- return matches[0];
60
+ return pickIntegrationAppForDatasourceIdentifier(datasourceKey, appNames);
59
61
  }
60
62
 
61
63
  /**
@@ -94,7 +96,7 @@ function collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey) {
94
96
  } catch {
95
97
  continue;
96
98
  }
97
- if (parsed && typeof parsed === 'object' && parsed.key === datasourceKey) {
99
+ if (datasourceIdentifierMatchesFile(datasourceKey, name, parsed)) {
98
100
  hits.push(full);
99
101
  }
100
102
  }
@@ -102,21 +104,12 @@ function collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey) {
102
104
  }
103
105
 
104
106
  /**
105
- * Find integration JSON whose parsed `key` equals datasourceKey.
107
+ * @param {string} appName
106
108
  * @param {string} datasourceKey
107
- * @returns {string} Absolute file path
109
+ * @returns {string}
108
110
  * @throws {Error}
109
111
  */
110
- function resolvePathFromIntegrationDatasourceKey(datasourceKey) {
111
- const appNames = pathsUtil.listIntegrationAppNames();
112
- const appName = pickIntegrationAppForDatasourceKey(datasourceKey, appNames);
113
- if (!appName) {
114
- throw new Error(
115
- `No integration/<app>/ folder matches datasource key "${datasourceKey}". ` +
116
- 'Pass a path to the JSON file, or use a key whose prefix matches an app under integration/ ' +
117
- '(e.g. app test-e2e-hubspot for key test-e2e-hubspot-users).'
118
- );
119
- }
112
+ function resolvePathFromIntegrationFolderScan(appName, datasourceKey) {
120
113
  const dir = pathsUtil.getIntegrationPath(appName);
121
114
  if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
122
115
  throw new Error(`Integration folder not found: ${dir}`);
@@ -128,16 +121,40 @@ function resolvePathFromIntegrationDatasourceKey(datasourceKey) {
128
121
  const hits = collectJsonPathsMatchingDatasourceKey(dir, scanOrder, datasourceKey);
129
122
  if (hits.length === 0) {
130
123
  throw new Error(
131
- `No datasource JSON with key "${datasourceKey}" under integration/${appName}/. ` +
124
+ `No datasource JSON matching "${datasourceKey}" under integration/${appName}/. ` +
132
125
  `Checked: ${scanOrder.length ? scanOrder.join(', ') : '(no JSON candidates)'}`
133
126
  );
134
127
  }
135
128
  if (hits.length > 1) {
136
- throw new Error(`Ambiguous: multiple files declare key "${datasourceKey}": ${hits.join(', ')}`);
129
+ throw new Error(`Ambiguous: multiple files match "${datasourceKey}": ${hits.join(', ')}`);
137
130
  }
138
131
  return hits[0];
139
132
  }
140
133
 
134
+ /**
135
+ * Find integration JSON whose parsed `key` equals datasourceKey.
136
+ * @param {string} datasourceKey
137
+ * @returns {string} Absolute file path
138
+ * @throws {Error}
139
+ */
140
+ function resolvePathFromIntegrationDatasourceKey(datasourceKey) {
141
+ const appNames = pathsUtil.listIntegrationAppNames();
142
+ const appName = pickIntegrationAppForDatasourceKey(datasourceKey, appNames);
143
+ if (!appName) {
144
+ throw new Error(
145
+ `No integration/<app>/ folder matches datasource identifier "${datasourceKey}". ` +
146
+ 'Pass a path to the JSON file, or use a datasource key or filename whose prefix matches an app under integration/ ' +
147
+ '(e.g. test-e2e-hubspot-companies or test-e2e-hubspot-datasource-companies for app test-e2e-hubspot).'
148
+ );
149
+ }
150
+ const manifestHits = collectDatasourceFileRefsMatchingIdentifier(appName, datasourceKey);
151
+ if (manifestHits.length === 1) {
152
+ const dir = pathsUtil.getIntegrationPath(appName);
153
+ return path.join(dir, manifestHits[0]);
154
+ }
155
+ return resolvePathFromIntegrationFolderScan(appName, datasourceKey);
156
+ }
157
+
141
158
  /**
142
159
  * @param {string} identifier - File path or datasource `key`
143
160
  * @returns {string} Absolute path to JSON file