@aifabrix/builder 2.40.2 → 2.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/.cursor/rules/docs-rules.mdc +30 -0
  2. package/README.md +7 -5
  3. package/integration/hubspot/README.md +8 -4
  4. package/integration/hubspot/application.json +54 -0
  5. package/integration/hubspot/create-hubspot.js +9 -136
  6. package/integration/hubspot/env.template +3 -4
  7. package/integration/hubspot/hubspot-datasource-company.json +343 -5
  8. package/integration/hubspot/hubspot-datasource-contact.json +413 -5
  9. package/integration/hubspot/hubspot-datasource-deal.json +341 -4
  10. package/integration/hubspot/hubspot-datasource-users.json +116 -0
  11. package/integration/hubspot/hubspot-deploy.json +1250 -108
  12. package/integration/hubspot/hubspot-system.json +15 -32
  13. package/integration/hubspot/test-dataplane-down-tests.js +17 -16
  14. package/integration/hubspot/test-dataplane-down.js +2 -2
  15. package/integration/hubspot/test.js +1 -1
  16. package/jest.config.manual.js +2 -1
  17. package/lib/api/credential.api.js +40 -0
  18. package/lib/api/dev.api.js +423 -0
  19. package/lib/api/external-test.api.js +111 -0
  20. package/lib/api/index.js +42 -19
  21. package/lib/api/pipeline.api.js +66 -120
  22. package/lib/api/types/credential.types.js +23 -0
  23. package/lib/api/types/dev.types.js +140 -0
  24. package/lib/api/types/pipeline.types.js +37 -0
  25. package/lib/api/wizard-platform.api.js +61 -0
  26. package/lib/api/wizard.api.js +34 -1
  27. package/lib/app/config.js +44 -11
  28. package/lib/app/down.js +2 -1
  29. package/lib/app/index.js +12 -1
  30. package/lib/app/prompts.js +44 -29
  31. package/lib/app/push.js +36 -12
  32. package/lib/app/readme.js +9 -6
  33. package/lib/app/run-env-compose.js +264 -0
  34. package/lib/app/run-helpers.js +121 -118
  35. package/lib/app/run.js +148 -28
  36. package/lib/app/show-display.js +1 -1
  37. package/lib/app/show.js +5 -2
  38. package/lib/build/index.js +11 -3
  39. package/lib/cli/setup-app.js +172 -15
  40. package/lib/cli/setup-credential-deployment.js +31 -6
  41. package/lib/cli/setup-dev.js +206 -16
  42. package/lib/cli/setup-environment.js +16 -6
  43. package/lib/cli/setup-external-system.js +89 -24
  44. package/lib/cli/setup-infra.js +82 -15
  45. package/lib/cli/setup-secrets.js +52 -5
  46. package/lib/cli/setup-utility.js +129 -24
  47. package/lib/commands/app-install.js +172 -0
  48. package/lib/commands/app-shell.js +75 -0
  49. package/lib/commands/app-test.js +282 -0
  50. package/lib/commands/app.js +1 -1
  51. package/lib/commands/credential-env.js +162 -0
  52. package/lib/commands/credential-list.js +17 -22
  53. package/lib/commands/credential-push.js +96 -0
  54. package/lib/commands/datasource.js +77 -6
  55. package/lib/commands/dev-cli-handlers.js +141 -0
  56. package/lib/commands/dev-down.js +114 -0
  57. package/lib/commands/dev-init.js +347 -0
  58. package/lib/commands/repair-auth-config.js +99 -0
  59. package/lib/commands/repair-datasource-keys.js +208 -0
  60. package/lib/commands/repair-datasource.js +235 -0
  61. package/lib/commands/repair-env-template.js +348 -0
  62. package/lib/commands/repair-internal.js +85 -0
  63. package/lib/commands/repair-rbac.js +158 -0
  64. package/lib/commands/repair.js +507 -0
  65. package/lib/commands/secrets-list.js +118 -0
  66. package/lib/commands/secrets-remove.js +97 -0
  67. package/lib/commands/secrets-set.js +30 -17
  68. package/lib/commands/secrets-validate.js +50 -0
  69. package/lib/commands/test-e2e-external.js +165 -0
  70. package/lib/commands/up-dataplane.js +2 -2
  71. package/lib/commands/up-miso.js +0 -25
  72. package/lib/commands/upload.js +96 -40
  73. package/lib/commands/wizard-core-helpers.js +226 -4
  74. package/lib/commands/wizard-core.js +67 -29
  75. package/lib/commands/wizard-dataplane.js +1 -1
  76. package/lib/commands/wizard-entity-selection.js +43 -0
  77. package/lib/commands/wizard-headless.js +44 -5
  78. package/lib/commands/wizard-helpers.js +7 -3
  79. package/lib/commands/wizard.js +86 -64
  80. package/lib/core/admin-secrets.js +96 -0
  81. package/lib/core/config.js +7 -1
  82. package/lib/core/secrets-ensure.js +378 -0
  83. package/lib/core/secrets-env-write.js +157 -0
  84. package/lib/core/secrets.js +176 -89
  85. package/lib/datasource/deploy.js +12 -3
  86. package/lib/datasource/field-reference-validator.js +91 -0
  87. package/lib/datasource/test-e2e.js +219 -0
  88. package/lib/datasource/test-integration.js +154 -0
  89. package/lib/datasource/validate.js +21 -3
  90. package/lib/deployment/deployer.js +7 -5
  91. package/lib/deployment/environment-config.js +137 -0
  92. package/lib/deployment/environment.js +21 -98
  93. package/lib/deployment/push.js +32 -2
  94. package/lib/external-system/download.js +188 -203
  95. package/lib/external-system/generator.js +204 -56
  96. package/lib/external-system/test-auth.js +7 -3
  97. package/lib/external-system/test-execution.js +2 -1
  98. package/lib/external-system/test-system-level.js +73 -0
  99. package/lib/external-system/test.js +56 -19
  100. package/lib/generator/external-controller-manifest.js +29 -2
  101. package/lib/generator/external-schema-utils.js +1 -1
  102. package/lib/generator/external.js +10 -3
  103. package/lib/generator/index.js +177 -25
  104. package/lib/generator/split-readme.js +1 -0
  105. package/lib/generator/split-variables.js +7 -1
  106. package/lib/generator/split.js +194 -54
  107. package/lib/generator/wizard-prompts-secondary.js +294 -0
  108. package/lib/generator/wizard-prompts.js +105 -106
  109. package/lib/generator/wizard-readme.js +88 -0
  110. package/lib/generator/wizard.js +155 -158
  111. package/lib/infrastructure/compose.js +11 -1
  112. package/lib/infrastructure/helpers.js +103 -20
  113. package/lib/infrastructure/index.js +98 -12
  114. package/lib/infrastructure/services.js +88 -22
  115. package/lib/schema/application-schema.json +32 -8
  116. package/lib/schema/external-datasource.schema.json +49 -26
  117. package/lib/schema/external-system.schema.json +509 -411
  118. package/lib/schema/wizard-config.schema.json +16 -0
  119. package/lib/utils/api.js +41 -13
  120. package/lib/utils/app-register-auth.js +25 -3
  121. package/lib/utils/auth-headers.js +8 -7
  122. package/lib/utils/cli-utils.js +20 -0
  123. package/lib/utils/compose-generator.js +77 -76
  124. package/lib/utils/compose-handlebars-helpers.js +54 -0
  125. package/lib/utils/compose-vector-helper.js +18 -0
  126. package/lib/utils/config-format-preference.js +51 -0
  127. package/lib/utils/config-format.js +36 -0
  128. package/lib/utils/config-paths.js +127 -2
  129. package/lib/utils/configuration-env-resolver.js +179 -0
  130. package/lib/utils/credential-display.js +83 -0
  131. package/lib/utils/credential-secrets-env.js +357 -0
  132. package/lib/utils/dataplane-pipeline-warning.js +28 -0
  133. package/lib/utils/deployment-validation-helpers.js +4 -4
  134. package/lib/utils/dev-ca-install.js +139 -0
  135. package/lib/utils/dev-cert-helper.js +122 -0
  136. package/lib/utils/device-code-helpers.js +224 -0
  137. package/lib/utils/device-code.js +37 -336
  138. package/lib/utils/docker-build.js +40 -8
  139. package/lib/utils/env-copy.js +103 -13
  140. package/lib/utils/env-map.js +35 -5
  141. package/lib/utils/env-template.js +6 -5
  142. package/lib/utils/error-formatters/http-status-errors.js +20 -2
  143. package/lib/utils/error-formatters/permission-errors.js +0 -1
  144. package/lib/utils/error-formatters/validation-errors.js +0 -1
  145. package/lib/utils/external-readme.js +56 -29
  146. package/lib/utils/external-system-display.js +59 -1
  147. package/lib/utils/external-system-test-helpers.js +21 -8
  148. package/lib/utils/external-system-validators.js +3 -0
  149. package/lib/utils/file-upload.js +20 -50
  150. package/lib/utils/help-builder.js +16 -2
  151. package/lib/utils/infra-status.js +80 -45
  152. package/lib/utils/local-secrets.js +7 -52
  153. package/lib/utils/mutagen-install.js +195 -0
  154. package/lib/utils/mutagen.js +146 -0
  155. package/lib/utils/paths.js +128 -37
  156. package/lib/utils/port-resolver.js +28 -16
  157. package/lib/utils/remote-dev-auth.js +38 -0
  158. package/lib/utils/remote-docker-env.js +43 -0
  159. package/lib/utils/remote-secrets-loader.js +60 -0
  160. package/lib/utils/secrets-canonical.js +93 -0
  161. package/lib/utils/secrets-generator.js +114 -6
  162. package/lib/utils/secrets-helpers.js +108 -114
  163. package/lib/utils/secrets-path.js +2 -2
  164. package/lib/utils/secrets-utils.js +52 -1
  165. package/lib/utils/secrets-validation.js +84 -0
  166. package/lib/utils/ssh-key-helper.js +116 -0
  167. package/lib/utils/test-log-writer.js +56 -0
  168. package/lib/utils/token-manager-messages.js +90 -0
  169. package/lib/utils/token-manager.js +29 -36
  170. package/lib/utils/variable-transformer.js +3 -3
  171. package/lib/validation/env-template-auth.js +157 -0
  172. package/lib/validation/env-template-kv.js +41 -0
  173. package/lib/validation/external-manifest-validator.js +25 -0
  174. package/lib/validation/external-system-auth-rules.js +86 -0
  175. package/lib/validation/validate-batch.js +149 -0
  176. package/lib/validation/validate-datasource-keys-api.js +33 -0
  177. package/lib/validation/validate-display.js +94 -16
  178. package/lib/validation/validate.js +25 -12
  179. package/lib/validation/validator.js +72 -9
  180. package/lib/validation/wizard-datasource-validation.js +50 -0
  181. package/package.json +8 -3
  182. package/scripts/install-local.js +34 -15
  183. package/templates/README.md +0 -1
  184. package/templates/applications/README.md.hbs +4 -4
  185. package/templates/applications/dataplane/application.yaml +6 -5
  186. package/templates/applications/dataplane/env.template +15 -10
  187. package/templates/applications/dataplane/rbac.yaml +2 -2
  188. package/templates/applications/keycloak/env.template +2 -0
  189. package/templates/applications/miso-controller/application.yaml +1 -0
  190. package/templates/applications/miso-controller/env.template +12 -10
  191. package/templates/external-system/README.md.hbs +65 -25
  192. package/templates/external-system/deploy.js.hbs +4 -2
  193. package/templates/external-system/external-datasource.yaml.hbs +217 -0
  194. package/templates/external-system/external-system.json.hbs +1 -18
  195. package/templates/infra/compose.yaml.hbs +6 -0
  196. package/templates/python/docker-compose.hbs +49 -23
  197. package/templates/typescript/docker-compose.hbs +48 -22
  198. package/integration/hubspot/application.yaml +0 -37
@@ -0,0 +1,158 @@
1
+ /**
2
+ * RBAC merge for repair: build permissions from datasources and ensure default roles.
3
+ *
4
+ * @fileoverview Repair RBAC merge from datasource resourceType/capabilities
5
+ * @author AI Fabrix Team
6
+ * @version 2.0.0
7
+ */
8
+
9
+ 'use strict';
10
+
11
+ const path = require('path');
12
+ const fs = require('fs');
13
+ const yaml = require('js-yaml');
14
+ const chalk = require('chalk');
15
+ const logger = require('../utils/logger');
16
+ const { loadConfigFile } = require('../utils/config-format');
17
+
18
+ const DEFAULT_CAPABILITIES = ['list', 'get', 'create', 'update', 'delete'];
19
+
20
+ /**
21
+ * Resolves capabilities from datasource (array or legacy object).
22
+ * @param {Object} parsed - Parsed datasource
23
+ * @returns {string[]}
24
+ */
25
+ function getCapabilitiesFromDatasource(parsed) {
26
+ const cap = parsed?.capabilities;
27
+ if (Array.isArray(cap)) return cap.filter(c => typeof c === 'string');
28
+ if (cap && typeof cap === 'object') {
29
+ return Object.keys(cap).filter(k => cap[k] === true);
30
+ }
31
+ return [...DEFAULT_CAPABILITIES];
32
+ }
33
+
34
+ /**
35
+ * Collects permission names (resourceType:capability) from datasource files.
36
+ * @param {string} appPath - Application path
37
+ * @param {string[]} datasourceFiles - Datasource file names
38
+ * @returns {Set<string>}
39
+ */
40
+ function collectPermissionNames(appPath, datasourceFiles) {
41
+ const permissionNames = new Set();
42
+ for (const fileName of datasourceFiles || []) {
43
+ const filePath = path.join(appPath, fileName);
44
+ if (!fs.existsSync(filePath)) continue;
45
+ try {
46
+ const parsed = loadConfigFile(filePath);
47
+ const resourceType = parsed?.resourceType || 'document';
48
+ const caps = getCapabilitiesFromDatasource(parsed);
49
+ for (const cap of caps) permissionNames.add(`${resourceType}:${cap}`);
50
+ } catch (err) {
51
+ logger.log(chalk.yellow(`⚠ Could not load ${fileName} for RBAC: ${err.message}`));
52
+ }
53
+ }
54
+ return permissionNames;
55
+ }
56
+
57
+ /**
58
+ * Loads existing rbac or creates empty structure. Uses extractRbacFromSystem when provided.
59
+ * @param {string} appPath - Application path
60
+ * @param {Object} [systemParsed] - Parsed system for extractRbacFromSystem
61
+ * @param {Function} extractRbacFromSystem - (system) => rbac or null
62
+ * @returns {{ rbac: Object, rbacPath: string, rbacYmlPath: string }}
63
+ */
64
+ function loadOrCreateRbac(appPath, systemParsed, extractRbacFromSystem) {
65
+ const rbacPath = path.join(appPath, 'rbac.yaml');
66
+ const rbacYmlPath = path.join(appPath, 'rbac.yml');
67
+ let rbac;
68
+ if (fs.existsSync(rbacPath)) {
69
+ rbac = loadConfigFile(rbacPath);
70
+ } else if (fs.existsSync(rbacYmlPath)) {
71
+ rbac = loadConfigFile(rbacYmlPath);
72
+ } else {
73
+ rbac = extractRbacFromSystem(systemParsed) || { roles: [], permissions: [] };
74
+ if (!Array.isArray(rbac.roles)) rbac.roles = [];
75
+ if (!Array.isArray(rbac.permissions)) rbac.permissions = [];
76
+ }
77
+ return { rbac, rbacPath, rbacYmlPath };
78
+ }
79
+
80
+ /**
81
+ * Adds missing permissions to rbac.permissions. Mutates rbac.
82
+ * @param {Object} rbac - RBAC object (mutated)
83
+ * @param {Set<string>} permissionNames - Desired permission names
84
+ * @param {string[]} changes - Array to append to
85
+ * @returns {boolean}
86
+ */
87
+ function addMissingPermissions(rbac, permissionNames, changes) {
88
+ const existing = new Set((rbac.permissions || []).map(p => p?.name).filter(Boolean));
89
+ let updated = false;
90
+ for (const name of permissionNames) {
91
+ if (existing.has(name)) continue;
92
+ rbac.permissions = rbac.permissions || [];
93
+ rbac.permissions.push({ name, roles: [], description: `Permission: ${name}` });
94
+ existing.add(name);
95
+ changes.push(`Added RBAC permission: ${name}`);
96
+ updated = true;
97
+ }
98
+ return updated;
99
+ }
100
+
101
+ /**
102
+ * Ensures default Admin and Reader roles if rbac.roles is empty. Mutates rbac.
103
+ * @param {Object} rbac - RBAC object (mutated)
104
+ * @param {string} systemKey - System key
105
+ * @param {string} displayName - Display name
106
+ * @param {string[]} changes - Array to append to
107
+ * @returns {boolean}
108
+ */
109
+ function ensureDefaultRoles(rbac, systemKey, displayName, changes) {
110
+ const hasRoles = Array.isArray(rbac.roles) && rbac.roles.length > 0;
111
+ if (hasRoles) return false;
112
+ const adminValue = `${systemKey}-admin`;
113
+ const readerValue = `${systemKey}-reader`;
114
+ rbac.roles = [
115
+ { name: `${displayName} Admin`, value: adminValue, description: `Full access to all ${displayName} operations`, groups: [] },
116
+ { name: `${displayName} Reader`, value: readerValue, description: `Read-only access to all ${displayName} data`, groups: [] }
117
+ ];
118
+ const listGetPerms = (rbac.permissions || [])
119
+ .filter(p => p?.name && (p.name.split(':')[1] === 'list' || p.name.split(':')[1] === 'get'))
120
+ .map(p => p.name);
121
+ for (const p of rbac.permissions || []) {
122
+ if (!p.roles) p.roles = [];
123
+ if (p.name && listGetPerms.includes(p.name) && !p.roles.includes(readerValue)) p.roles.push(readerValue);
124
+ if (!p.roles.includes(adminValue)) p.roles.push(adminValue);
125
+ }
126
+ changes.push('Added default Admin and Reader roles to rbac.yaml');
127
+ return true;
128
+ }
129
+
130
+ /**
131
+ * Merges RBAC from datasources: ensures permission per resourceType:capability, adds Admin/Reader roles if none.
132
+ * @param {string} appPath - Application path
133
+ * @param {Object} systemParsed - Parsed system (key, displayName)
134
+ * @param {string[]} datasourceFiles - Datasource file names
135
+ * @param {Function} extractRbacFromSystem - (system) => rbac or null
136
+ * @param {boolean} dryRun - If true, do not write
137
+ * @param {string[]} changes - Array to append change descriptions to
138
+ * @returns {boolean} True if rbac was updated (or would be in dry-run)
139
+ */
140
+ function mergeRbacFromDatasources(appPath, systemParsed, datasourceFiles, extractRbacFromSystem, dryRun, changes) {
141
+ const permissionNames = collectPermissionNames(appPath, datasourceFiles);
142
+ if (permissionNames.size === 0) return false;
143
+ const systemKey = systemParsed?.key || 'system';
144
+ const displayName = systemParsed?.displayName || systemKey;
145
+ const { rbac, rbacPath, rbacYmlPath } = loadOrCreateRbac(appPath, systemParsed, extractRbacFromSystem);
146
+ let updated = addMissingPermissions(rbac, permissionNames, changes);
147
+ updated = ensureDefaultRoles(rbac, systemKey, displayName, changes) || updated;
148
+ if (updated && !dryRun) {
149
+ const outPath = fs.existsSync(rbacPath) ? rbacPath : (fs.existsSync(rbacYmlPath) ? rbacYmlPath : rbacPath);
150
+ fs.writeFileSync(outPath, yaml.dump(rbac, { indent: 2, lineWidth: -1 }), { mode: 0o644, encoding: 'utf8' });
151
+ }
152
+ return updated;
153
+ }
154
+
155
+ module.exports = {
156
+ getCapabilitiesFromDatasource,
157
+ mergeRbacFromDatasources
158
+ };
@@ -0,0 +1,507 @@
1
+ /**
2
+ * Repair external integration config: fix drift between config and files on disk.
3
+ *
4
+ * Aligns application.yaml with actual system/datasource files, fixes system key mismatch,
5
+ * creates missing externalIntegration block, extracts rbac.yaml from system when needed,
6
+ * and regenerates the deployment manifest.
7
+ *
8
+ * @fileoverview Repair external integration drift
9
+ * @author AI Fabrix Team
10
+ * @version 2.0.0
11
+ */
12
+ /* eslint-disable max-lines -- Repair flow with auth, normalization, and steps */
13
+
14
+ 'use strict';
15
+
16
+ const path = require('path');
17
+ const fs = require('fs');
18
+ const chalk = require('chalk');
19
+ const yaml = require('js-yaml');
20
+ const { detectAppType } = require('../utils/paths');
21
+ const { resolveApplicationConfigPath } = require('../utils/app-config-resolver');
22
+ const { loadConfigFile, writeConfigFile, writeYamlPreservingComments, isYamlPath } = require('../utils/config-format');
23
+ const { systemKeyToKvPrefix, securityKeyToVar } = require('../utils/credential-secrets-env');
24
+ const logger = require('../utils/logger');
25
+ const generator = require('../generator');
26
+ const { repairEnvTemplate, normalizeSystemFileAuthAndConfig } = require('./repair-env-template');
27
+ const { repairDatasourceFile } = require('./repair-datasource');
28
+ const { mergeRbacFromDatasources } = require('./repair-rbac');
29
+ const { discoverIntegrationFiles, buildEffectiveDatasourceFiles } = require('./repair-internal');
30
+ const { normalizeDatasourceKeysAndFilenames } = require('./repair-datasource-keys');
31
+
32
+ /** Allowed authentication methods for repair --auth (matches external-system schema) */
33
+ const ALLOWED_AUTH = ['oauth2', 'aad', 'apikey', 'basic', 'queryParam', 'oidc', 'hmac', 'none'];
34
+
35
+ /**
36
+ * Extracts roles and permissions from system object for rbac.yaml
37
+ * @param {Object} system - Parsed system config
38
+ * @returns {Object|null} RBAC object or null
39
+ */
40
+ function extractRbacFromSystem(system) {
41
+ if (!system || typeof system !== 'object') return null;
42
+ const hasRoles = system.roles && Array.isArray(system.roles) && system.roles.length > 0;
43
+ const hasPermissions = system.permissions && Array.isArray(system.permissions) && system.permissions.length > 0;
44
+ if (!hasRoles && !hasPermissions) return null;
45
+ const rbac = {};
46
+ if (hasRoles) rbac.roles = system.roles;
47
+ if (hasPermissions) rbac.permissions = system.permissions;
48
+ return rbac;
49
+ }
50
+
51
+ /**
52
+ * Loads first system file and returns parsed object with key
53
+ * @param {string} appPath - Application path
54
+ * @param {string} systemFileName - System file name
55
+ * @returns {Object} Parsed system config
56
+ */
57
+ function loadFirstSystemFile(appPath, systemFileName) {
58
+ const systemPath = path.join(appPath, systemFileName);
59
+ if (!fs.existsSync(systemPath)) {
60
+ throw new Error(`System file not found: ${systemPath}`);
61
+ }
62
+ return loadConfigFile(systemPath);
63
+ }
64
+
65
+ function resolveSystemContext(appPath, systemFiles) {
66
+ const systemFilePath = path.join(appPath, systemFiles[0]);
67
+ const systemParsed = loadFirstSystemFile(appPath, systemFiles[0]);
68
+ const systemKey = systemParsed.key ||
69
+ path.basename(systemFiles[0], path.extname(systemFiles[0])).replace(/-system$/, '');
70
+ return { systemFilePath, systemParsed, systemKey };
71
+ }
72
+
73
+ function ensureExternalIntegrationBlock(variables, systemFiles, datasourceFiles, changes) {
74
+ const extInt = variables.externalIntegration;
75
+ let updated = false;
76
+ if (!extInt) {
77
+ variables.externalIntegration = {
78
+ schemaBasePath: './',
79
+ systems: systemFiles,
80
+ dataSources: datasourceFiles,
81
+ autopublish: true,
82
+ version: '1.0.0'
83
+ };
84
+ changes.push('Created externalIntegration block from discovered files');
85
+ updated = true;
86
+ } else {
87
+ const prevSystems = extInt.systems || [];
88
+ const prevDataSources = extInt.dataSources || [];
89
+ if (JSON.stringify(prevSystems) !== JSON.stringify(systemFiles)) {
90
+ changes.push(`systems: [${prevSystems.join(', ')}] → [${systemFiles.join(', ')}]`);
91
+ extInt.systems = systemFiles;
92
+ updated = true;
93
+ }
94
+ if (JSON.stringify(prevDataSources) !== JSON.stringify(datasourceFiles)) {
95
+ changes.push(`dataSources: [${prevDataSources.join(', ')}] → [${datasourceFiles.join(', ')}]`);
96
+ extInt.dataSources = datasourceFiles;
97
+ updated = true;
98
+ }
99
+ }
100
+ return updated;
101
+ }
102
+
103
+ function alignAppKeyWithSystem(variables, systemKey, systemParsed, changes) {
104
+ const appKey = variables.app?.key;
105
+ if (!appKey || appKey === systemKey) return false;
106
+ if (!variables.app) variables.app = {};
107
+ variables.app.key = systemKey;
108
+ changes.push(`app.key: ${appKey} → ${systemKey}`);
109
+ return true;
110
+ }
111
+
112
+ /**
113
+ * Aligns datasource systemKey values to match the system key.
114
+ * Updates each datasource file whose systemKey differs from the system key.
115
+ *
116
+ * @param {string} appPath - Application directory path
117
+ * @param {string[]} datasourceFiles - Datasource file names
118
+ * @param {string} systemKey - Expected system key
119
+ * @param {boolean} dryRun - If true, report changes but do not write
120
+ * @param {string[]} changes - Array to append change descriptions to
121
+ * @returns {boolean} True if any file was updated (or would be in dry-run)
122
+ */
123
+ function alignDatasourceSystemKeys(appPath, datasourceFiles, systemKey, dryRun, changes) {
124
+ if (!datasourceFiles || datasourceFiles.length === 0) return false;
125
+ let updated = false;
126
+ for (const datasourceFile of datasourceFiles) {
127
+ const datasourcePath = path.join(appPath, datasourceFile);
128
+ if (!fs.existsSync(datasourcePath)) continue;
129
+ const parsed = loadConfigFile(datasourcePath);
130
+ const old = parsed.systemKey;
131
+ if (old !== systemKey) {
132
+ parsed.systemKey = systemKey;
133
+ if (!dryRun) {
134
+ writeConfigFile(datasourcePath, parsed);
135
+ }
136
+ changes.push(`${datasourceFile}: systemKey ${old} → ${systemKey}`);
137
+ updated = true;
138
+ }
139
+ }
140
+ return updated;
141
+ }
142
+
143
+ /**
144
+ * Derives a datasource key from filename when the file has no key.
145
+ * - hubspot-datasource-company.json → hubspot-company
146
+ * - datasource-companies.json → {systemKey}-companies (e.g. test-hubspot-companies)
147
+ *
148
+ * @param {string} fileName - Datasource file name
149
+ * @param {string} systemKey - System key (e.g. test-hubspot), used for datasource-*.json style names
150
+ * @returns {string}
151
+ */
152
+ function deriveDatasourceKeyFromFileName(fileName, systemKey) {
153
+ const base = path.basename(fileName, path.extname(fileName));
154
+ if (/^datasource-/.test(base)) {
155
+ const suffix = base.slice('datasource-'.length);
156
+ return systemKey && typeof systemKey === 'string' ? `${systemKey}-${suffix}` : base;
157
+ }
158
+ return base.replace(/-datasource-/, '-');
159
+ }
160
+
161
+ /**
162
+ * Aligns system file dataSources array to match datasource keys from discovered files.
163
+ * The system file holds logical keys (not filenames): each key comes from that datasource
164
+ * file's "key" property, or is derived from the filename when missing (e.g. datasource-companies.json → {systemKey}-companies).
165
+ *
166
+ * @param {string} appPath - Application directory path
167
+ * @param {Object} systemParsed - Parsed system config (mutated)
168
+ * @param {string[]} datasourceFiles - Datasource file names
169
+ * @param {string} systemKey - System key for deriving key when missing
170
+ * @param {boolean} dryRun - If true, report changes but do not write
171
+ * @param {string[]} changes - Array to append change descriptions to
172
+ * @returns {boolean} True if dataSources was updated (or would be in dry-run)
173
+ */
174
+ function alignSystemFileDataSources(appPath, systemParsed, datasourceFiles, systemKey, dryRun, changes) {
175
+ const keys = [];
176
+ for (const fileName of datasourceFiles) {
177
+ const filePath = path.join(appPath, fileName);
178
+ if (!fs.existsSync(filePath)) continue;
179
+ try {
180
+ const parsed = loadConfigFile(filePath);
181
+ const key = parsed && typeof parsed.key === 'string' && parsed.key.trim()
182
+ ? parsed.key.trim()
183
+ : deriveDatasourceKeyFromFileName(fileName, systemKey);
184
+ keys.push(key);
185
+ } catch (err) {
186
+ logger.log(chalk.yellow(`⚠ Could not load datasource file ${fileName}: ${err.message}; using derived key`));
187
+ keys.push(deriveDatasourceKeyFromFileName(fileName, systemKey));
188
+ }
189
+ }
190
+ keys.sort();
191
+ const prev = Array.isArray(systemParsed.dataSources) ? [...systemParsed.dataSources].sort() : [];
192
+ if (JSON.stringify(prev) === JSON.stringify(keys)) return false;
193
+ systemParsed.dataSources = keys;
194
+ changes.push(`dataSources: [${prev.join(', ') || '(none)'}] → [${keys.join(', ')}] (keys from each datasource file's "key" or filename)`);
195
+ return true;
196
+ }
197
+
198
+ /**
199
+ * Builds the set of auth variable names (UPPERCASE, no underscores) from authentication.variables and authentication.security.
200
+ * Used to detect configuration entries that belong only in the authentication section.
201
+ * Canonical keys per method are in lib/schema/external-system.schema.json $defs.authenticationVariablesByMethod
202
+ * (e.g. oauth2: variables baseUrl, tokenUrl, scope; security clientId, clientSecret).
203
+ *
204
+ * @param {Object} systemParsed - Parsed system config
205
+ * @param {string} _systemKey - System key (unused; for API consistency)
206
+ * @returns {Set<string>}
207
+ */
208
+ function buildAuthVarNames(systemParsed, _systemKey) {
209
+ const names = new Set();
210
+ const auth = systemParsed.authentication;
211
+ if (auth && typeof auth.variables === 'object') {
212
+ for (const k of Object.keys(auth.variables)) {
213
+ names.add(String(k).toUpperCase().replace(/_/g, ''));
214
+ }
215
+ }
216
+ if (auth && typeof auth.security === 'object') {
217
+ for (const k of Object.keys(auth.security)) {
218
+ names.add(securityKeyToVar(k));
219
+ }
220
+ }
221
+ return names;
222
+ }
223
+
224
+ /**
225
+ * Removes from system configuration any entry that represents standard auth variables
226
+ * (BASEURL, CLIENTID, CLIENTSECRET, TOKENURL, APIKEY, USERNAME, PASSWORD, etc.).
227
+ * These are supplied from the selected credential at runtime; the configuration array
228
+ * should contain only custom variables. Removes both plain and keyvault auth entries.
229
+ *
230
+ * @param {Object} systemParsed - Parsed system config (mutated)
231
+ * @param {string} systemKey - System key for naming consistency
232
+ * @param {boolean} dryRun - If true, report changes but do not write
233
+ * @param {string[]} changes - Array to append change descriptions to
234
+ * @returns {boolean} True if any entry was removed
235
+ */
236
+ /**
237
+ * Derives the normalized auth variable part from a config entry name (for matching against authNames).
238
+ * E.g. KV_HUBSPOT_CLIENTID → CLIENTID, BASEURL → BASEURL.
239
+ * @param {string} name - Entry name
240
+ * @param {string} systemKey - System key for KV_ prefix
241
+ * @returns {string}
242
+ */
243
+ function normalizedAuthPartFromConfigName(name, systemKey) {
244
+ const n = String(name).trim();
245
+ if (!n) return '';
246
+ const prefix = systemKeyToKvPrefix(systemKey);
247
+ const kvPrefix = `KV_${prefix}_`;
248
+ if (n.toUpperCase().startsWith(kvPrefix)) {
249
+ const rest = n.slice(kvPrefix.length);
250
+ return rest.toUpperCase().replace(/_/g, '');
251
+ }
252
+ return n.toUpperCase().replace(/_/g, '');
253
+ }
254
+
255
+ function removeAuthVarsFromConfiguration(systemParsed, systemKey, dryRun, changes) {
256
+ const config = systemParsed.configuration;
257
+ if (!Array.isArray(config)) return false;
258
+ const authNames = buildAuthVarNames(systemParsed, systemKey);
259
+ if (authNames.size === 0) return false;
260
+ const removed = [];
261
+ const filtered = config.filter((entry) => {
262
+ if (!entry || !entry.name) return true;
263
+ const authPart = normalizedAuthPartFromConfigName(entry.name, systemKey);
264
+ if (authNames.has(authPart)) {
265
+ removed.push(entry.name);
266
+ return false;
267
+ }
268
+ return true;
269
+ });
270
+ if (removed.length === 0) return false;
271
+ systemParsed.configuration = filtered;
272
+ changes.push(`Removed authentication variable(s) from configuration: ${removed.join(', ')}`);
273
+ return true;
274
+ }
275
+
276
+ function createRbacFromSystemIfNeeded(appPath, systemFilePath, systemParsed, dryRun, changes) {
277
+ const rbacPath = path.join(appPath, 'rbac.yaml');
278
+ const rbacYmlPath = path.join(appPath, 'rbac.yml');
279
+ if (fs.existsSync(rbacPath) || fs.existsSync(rbacYmlPath)) return false;
280
+ const rbacFromSystem = extractRbacFromSystem(systemParsed);
281
+ if (!rbacFromSystem) return false;
282
+ if (!dryRun) {
283
+ const rbacYaml = yaml.dump(rbacFromSystem, { indent: 2, lineWidth: -1 });
284
+ fs.writeFileSync(rbacPath, rbacYaml, { mode: 0o644, encoding: 'utf8' });
285
+ delete systemParsed.roles;
286
+ delete systemParsed.permissions;
287
+ writeConfigFile(systemFilePath, systemParsed);
288
+ }
289
+ changes.push('Created rbac.yaml from system roles/permissions');
290
+ changes.push('Removed roles/permissions from system file (now in rbac.yaml)');
291
+ return true;
292
+ }
293
+
294
+ /**
295
+ * Runs datasource repair for each file (dimensions, metadataSchema, optional expose/sync/test).
296
+ * @param {string} appPath - Application path
297
+ * @param {string[]} datasourceFiles - Datasource file names
298
+ * @param {Object} options - { expose?: boolean, sync?: boolean, test?: boolean }
299
+ * @param {boolean} dryRun - If true, do not write
300
+ * @param {string[]} changes - Array to append change descriptions to
301
+ * @returns {boolean} True if any datasource was updated
302
+ */
303
+ function runDatasourceRepairs(appPath, datasourceFiles, options, dryRun, changes) {
304
+ if (!datasourceFiles || datasourceFiles.length === 0) return false;
305
+ let updated = false;
306
+ for (const fileName of datasourceFiles) {
307
+ const filePath = path.join(appPath, fileName);
308
+ if (!fs.existsSync(filePath)) continue;
309
+ try {
310
+ const parsed = loadConfigFile(filePath);
311
+ const { updated: fileUpdated, changes: fileChanges } = repairDatasourceFile(parsed, {
312
+ expose: options.expose,
313
+ sync: options.sync,
314
+ test: options.test
315
+ });
316
+ if (fileUpdated) {
317
+ updated = true;
318
+ fileChanges.forEach(c => changes.push(`${fileName}: ${c}`));
319
+ if (!dryRun) {
320
+ writeConfigFile(filePath, parsed);
321
+ }
322
+ }
323
+ } catch (err) {
324
+ logger.log(chalk.yellow(`⚠ Could not repair datasource ${fileName}: ${err.message}`));
325
+ }
326
+ }
327
+ return updated;
328
+ }
329
+
330
+ async function regenerateManifest(appName, appPath, changes) {
331
+ try {
332
+ const deployPath = await generator.generateDeployJson(appName, { appPath });
333
+ changes.push(`Regenerated ${path.basename(deployPath)}`);
334
+ return true;
335
+ } catch (err) {
336
+ logger.log(chalk.yellow(`⚠ Manifest regeneration skipped: ${err.message}`));
337
+ return false;
338
+ }
339
+ }
340
+
341
+ function persistChangesAndRegenerate(configPath, variables, appName, appPath, changes, originalYamlContent) {
342
+ if (originalYamlContent !== null && originalYamlContent !== undefined && typeof originalYamlContent === 'string' && isYamlPath(configPath)) {
343
+ writeYamlPreservingComments(configPath, originalYamlContent, variables);
344
+ } else {
345
+ writeConfigFile(configPath, variables);
346
+ }
347
+ logger.log(chalk.green(`✓ Updated ${path.basename(configPath)}`));
348
+ changes.forEach(c => logger.log(chalk.gray(` ${c}`)));
349
+ return regenerateManifest(appName, appPath, changes);
350
+ }
351
+
352
+ /**
353
+ * Apply --auth: replace system file authentication with canonical block for the given method.
354
+ * @param {Object} ctx - Context with systemParsed, systemKey, auth, dryRun, changes
355
+ * @returns {boolean} True if auth was replaced
356
+ */
357
+ function applyAuthMethod(ctx) {
358
+ if (!ctx.auth || typeof ctx.auth !== 'string') return false;
359
+ const method = ctx.auth.trim().toLowerCase();
360
+ if (!ALLOWED_AUTH.includes(method)) {
361
+ throw new Error(
362
+ `Invalid --auth "${ctx.auth}". Allowed methods: ${ALLOWED_AUTH.join(', ')}`
363
+ );
364
+ }
365
+ const { buildAuthenticationFromMethod } = require('../external-system/generator');
366
+ ctx.systemParsed.authentication = buildAuthenticationFromMethod(ctx.systemKey, method);
367
+ ctx.changes.push(`Set authentication method to ${method}`);
368
+ return true;
369
+ }
370
+
371
+ /**
372
+ * Runs all repair steps (integration block, system dataSources, auth/config, app key, datasource keys, rbac, env.template).
373
+ * @param {Object} ctx - Context with appPath, configPath, variables, systemFilePath, systemParsed, systemKey, systemFiles, datasourceFiles, dryRun, changes, auth?
374
+ * @returns {{ updated: boolean, appKeyFixed: boolean, datasourceKeysFixed: boolean, rbacFileCreated: boolean, envTemplateRepaired: boolean }}
375
+ */
376
+ function runRepairSteps(ctx) {
377
+ let updated = ensureExternalIntegrationBlock(
378
+ ctx.variables, ctx.systemFiles, ctx.datasourceFiles, ctx.changes
379
+ );
380
+ const authReplaced = applyAuthMethod(ctx);
381
+ updated = updated || authReplaced;
382
+ const systemAuthConfigNormalized = normalizeSystemFileAuthAndConfig(
383
+ ctx.systemParsed, ctx.systemKey, ctx.changes
384
+ );
385
+ const systemDataSourcesAligned = alignSystemFileDataSources(
386
+ ctx.appPath, ctx.systemParsed, ctx.datasourceFiles, ctx.systemKey, ctx.dryRun, ctx.changes
387
+ );
388
+ const authVarsRemoved = removeAuthVarsFromConfiguration(
389
+ ctx.systemParsed, ctx.systemKey, ctx.dryRun, ctx.changes
390
+ );
391
+ if ((authReplaced || systemAuthConfigNormalized || systemDataSourcesAligned || authVarsRemoved) && !ctx.dryRun) {
392
+ writeConfigFile(ctx.systemFilePath, ctx.systemParsed);
393
+ }
394
+ updated = updated || systemAuthConfigNormalized || systemDataSourcesAligned || authVarsRemoved;
395
+ const appKeyFixed = alignAppKeyWithSystem(
396
+ ctx.variables, ctx.systemKey, ctx.systemParsed, ctx.changes
397
+ );
398
+ const datasourceKeysFixed = alignDatasourceSystemKeys(
399
+ ctx.appPath, ctx.datasourceFiles, ctx.systemKey, ctx.dryRun, ctx.changes
400
+ );
401
+ const rbacFileCreated = createRbacFromSystemIfNeeded(
402
+ ctx.appPath, ctx.systemFilePath, ctx.systemParsed, ctx.dryRun, ctx.changes
403
+ );
404
+ const envTemplateRepaired = repairEnvTemplate(
405
+ ctx.appPath, ctx.systemParsed, ctx.systemKey, ctx.dryRun, ctx.changes
406
+ );
407
+ updated = updated || appKeyFixed || datasourceKeysFixed || rbacFileCreated || envTemplateRepaired;
408
+ return {
409
+ updated,
410
+ appKeyFixed,
411
+ datasourceKeysFixed,
412
+ rbacFileCreated,
413
+ envTemplateRepaired
414
+ };
415
+ }
416
+
417
+ /**
418
+ * Repairs external integration config: syncs files, aligns keys, creates rbac, regenerates manifest
419
+ * @async
420
+ * @param {string} appName - Application/integration name
421
+ * @param {Object} [options] - Options
422
+ * @param {boolean} [options.dryRun] - If true, only report changes; do not write
423
+ * @returns {Promise<{ updated: boolean, changes: string[], systemFiles: string[], datasourceFiles: string[], appKeyFixed?: boolean, datasourceKeysFixed?: boolean, rbacFileCreated?: boolean, envTemplateRepaired?: boolean, manifestRegenerated?: boolean }>}
424
+ */
425
+ /**
426
+ * Loads application config and discovers integration files; validates at least one system file exists.
427
+ * @param {string} appPath - Application path
428
+ * @param {string} configPath - Path to application config
429
+ * @returns {{ variables: Object, originalYamlContent: string|null, systemFiles: string[], datasourceFiles: string[] }}
430
+ */
431
+ function loadConfigAndDiscover(appPath, configPath) {
432
+ const variables = loadConfigFile(configPath);
433
+ let originalYamlContent = null;
434
+ if (isYamlPath(configPath)) {
435
+ originalYamlContent = fs.readFileSync(configPath, 'utf8');
436
+ }
437
+ const { systemFiles, datasourceFiles: discoveredDatasourceFiles } = discoverIntegrationFiles(appPath);
438
+ if (systemFiles.length === 0) {
439
+ throw new Error(`No system file found in ${appPath}. Expected *-system.yaml or *-system.json`);
440
+ }
441
+ const datasourceFiles = buildEffectiveDatasourceFiles(appPath, discoveredDatasourceFiles, variables.externalIntegration?.dataSources);
442
+ return { variables, originalYamlContent, systemFiles, datasourceFiles };
443
+ }
444
+
445
+ async function repairExternalIntegration(appName, options = {}) {
446
+ if (!appName || typeof appName !== 'string') throw new Error('App name is required');
447
+ const { dryRun = false, auth: authOption } = options;
448
+ if (authOption !== undefined && authOption !== null && typeof authOption !== 'string') {
449
+ throw new Error('Option --auth must be a string');
450
+ }
451
+ const { appPath, isExternal } = await detectAppType(appName);
452
+ if (!isExternal) throw new Error(`App '${appName}' is not an external integration`);
453
+ const configPath = resolveApplicationConfigPath(appPath);
454
+ if (!fs.existsSync(configPath)) throw new Error(`Application config not found: ${configPath}`);
455
+
456
+ const { variables, originalYamlContent, systemFiles, datasourceFiles: initialDatasourceFiles } = loadConfigAndDiscover(appPath, configPath);
457
+ const changes = [];
458
+ const { systemFilePath, systemParsed, systemKey } = resolveSystemContext(appPath, systemFiles);
459
+ const { updated: keysNormalized, datasourceFiles } = normalizeDatasourceKeysAndFilenames(
460
+ appPath,
461
+ initialDatasourceFiles,
462
+ systemKey,
463
+ variables,
464
+ dryRun,
465
+ changes
466
+ );
467
+ const steps = runRepairSteps({
468
+ appPath,
469
+ configPath,
470
+ variables,
471
+ systemFilePath,
472
+ systemParsed,
473
+ systemKey,
474
+ systemFiles,
475
+ datasourceFiles,
476
+ dryRun,
477
+ changes,
478
+ auth: authOption
479
+ });
480
+
481
+ const opts = { expose: Boolean(options.expose), sync: Boolean(options.sync), test: Boolean(options.test) };
482
+ const datasourceRepairUpdated = runDatasourceRepairs(appPath, datasourceFiles, opts, dryRun, changes);
483
+ const rbacMergeUpdated = options.rbac
484
+ ? mergeRbacFromDatasources(appPath, systemParsed, datasourceFiles, extractRbacFromSystem, dryRun, changes)
485
+ : false;
486
+ const anyUpdated = keysNormalized || steps.updated || datasourceRepairUpdated || rbacMergeUpdated;
487
+ const manifestRegenerated = (anyUpdated && !dryRun)
488
+ ? await persistChangesAndRegenerate(configPath, variables, appName, appPath, changes, originalYamlContent)
489
+ : false;
490
+
491
+ return {
492
+ updated: anyUpdated,
493
+ changes,
494
+ systemFiles,
495
+ datasourceFiles,
496
+ appKeyFixed: steps.appKeyFixed,
497
+ datasourceKeysFixed: steps.datasourceKeysFixed,
498
+ rbacFileCreated: steps.rbacFileCreated,
499
+ envTemplateRepaired: steps.envTemplateRepaired,
500
+ manifestRegenerated
501
+ };
502
+ }
503
+
504
+ module.exports = {
505
+ repairExternalIntegration,
506
+ discoverIntegrationFiles
507
+ };