@aifabrix/builder 2.40.2 → 2.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/.cursor/rules/docs-rules.mdc +30 -0
  2. package/README.md +7 -5
  3. package/integration/hubspot/README.md +8 -4
  4. package/integration/hubspot/application.json +54 -0
  5. package/integration/hubspot/create-hubspot.js +9 -136
  6. package/integration/hubspot/env.template +3 -4
  7. package/integration/hubspot/hubspot-datasource-company.json +343 -5
  8. package/integration/hubspot/hubspot-datasource-contact.json +413 -5
  9. package/integration/hubspot/hubspot-datasource-deal.json +341 -4
  10. package/integration/hubspot/hubspot-datasource-users.json +116 -0
  11. package/integration/hubspot/hubspot-deploy.json +1250 -108
  12. package/integration/hubspot/hubspot-system.json +15 -32
  13. package/integration/hubspot/test-dataplane-down-tests.js +17 -16
  14. package/integration/hubspot/test-dataplane-down.js +2 -2
  15. package/integration/hubspot/test.js +1 -1
  16. package/jest.config.manual.js +2 -1
  17. package/lib/api/credential.api.js +40 -0
  18. package/lib/api/dev.api.js +423 -0
  19. package/lib/api/external-test.api.js +111 -0
  20. package/lib/api/index.js +42 -19
  21. package/lib/api/pipeline.api.js +66 -120
  22. package/lib/api/types/credential.types.js +23 -0
  23. package/lib/api/types/dev.types.js +140 -0
  24. package/lib/api/types/pipeline.types.js +37 -0
  25. package/lib/api/wizard-platform.api.js +61 -0
  26. package/lib/api/wizard.api.js +34 -1
  27. package/lib/app/config.js +44 -11
  28. package/lib/app/down.js +2 -1
  29. package/lib/app/index.js +12 -1
  30. package/lib/app/prompts.js +44 -29
  31. package/lib/app/push.js +36 -12
  32. package/lib/app/readme.js +9 -6
  33. package/lib/app/run-env-compose.js +264 -0
  34. package/lib/app/run-helpers.js +121 -118
  35. package/lib/app/run.js +148 -28
  36. package/lib/app/show-display.js +1 -1
  37. package/lib/app/show.js +5 -2
  38. package/lib/build/index.js +11 -3
  39. package/lib/cli/setup-app.js +172 -15
  40. package/lib/cli/setup-credential-deployment.js +31 -6
  41. package/lib/cli/setup-dev.js +206 -16
  42. package/lib/cli/setup-environment.js +16 -6
  43. package/lib/cli/setup-external-system.js +89 -24
  44. package/lib/cli/setup-infra.js +82 -15
  45. package/lib/cli/setup-secrets.js +52 -5
  46. package/lib/cli/setup-utility.js +129 -24
  47. package/lib/commands/app-install.js +172 -0
  48. package/lib/commands/app-shell.js +75 -0
  49. package/lib/commands/app-test.js +282 -0
  50. package/lib/commands/app.js +1 -1
  51. package/lib/commands/credential-env.js +162 -0
  52. package/lib/commands/credential-list.js +17 -22
  53. package/lib/commands/credential-push.js +96 -0
  54. package/lib/commands/datasource.js +77 -6
  55. package/lib/commands/dev-cli-handlers.js +141 -0
  56. package/lib/commands/dev-down.js +114 -0
  57. package/lib/commands/dev-init.js +347 -0
  58. package/lib/commands/repair-auth-config.js +99 -0
  59. package/lib/commands/repair-datasource-keys.js +208 -0
  60. package/lib/commands/repair-datasource.js +235 -0
  61. package/lib/commands/repair-env-template.js +348 -0
  62. package/lib/commands/repair-internal.js +85 -0
  63. package/lib/commands/repair-rbac.js +158 -0
  64. package/lib/commands/repair.js +507 -0
  65. package/lib/commands/secrets-list.js +118 -0
  66. package/lib/commands/secrets-remove.js +97 -0
  67. package/lib/commands/secrets-set.js +30 -17
  68. package/lib/commands/secrets-validate.js +50 -0
  69. package/lib/commands/test-e2e-external.js +165 -0
  70. package/lib/commands/up-dataplane.js +2 -2
  71. package/lib/commands/up-miso.js +0 -25
  72. package/lib/commands/upload.js +96 -40
  73. package/lib/commands/wizard-core-helpers.js +226 -4
  74. package/lib/commands/wizard-core.js +67 -29
  75. package/lib/commands/wizard-dataplane.js +1 -1
  76. package/lib/commands/wizard-entity-selection.js +43 -0
  77. package/lib/commands/wizard-headless.js +44 -5
  78. package/lib/commands/wizard-helpers.js +7 -3
  79. package/lib/commands/wizard.js +86 -64
  80. package/lib/core/admin-secrets.js +96 -0
  81. package/lib/core/config.js +7 -1
  82. package/lib/core/secrets-ensure.js +378 -0
  83. package/lib/core/secrets-env-write.js +157 -0
  84. package/lib/core/secrets.js +176 -89
  85. package/lib/datasource/deploy.js +12 -3
  86. package/lib/datasource/field-reference-validator.js +91 -0
  87. package/lib/datasource/test-e2e.js +219 -0
  88. package/lib/datasource/test-integration.js +154 -0
  89. package/lib/datasource/validate.js +21 -3
  90. package/lib/deployment/deployer.js +7 -5
  91. package/lib/deployment/environment-config.js +137 -0
  92. package/lib/deployment/environment.js +21 -98
  93. package/lib/deployment/push.js +32 -2
  94. package/lib/external-system/download.js +188 -203
  95. package/lib/external-system/generator.js +204 -56
  96. package/lib/external-system/test-auth.js +7 -3
  97. package/lib/external-system/test-execution.js +2 -1
  98. package/lib/external-system/test-system-level.js +73 -0
  99. package/lib/external-system/test.js +56 -19
  100. package/lib/generator/external-controller-manifest.js +29 -2
  101. package/lib/generator/external-schema-utils.js +1 -1
  102. package/lib/generator/external.js +10 -3
  103. package/lib/generator/index.js +177 -25
  104. package/lib/generator/split-readme.js +1 -0
  105. package/lib/generator/split-variables.js +7 -1
  106. package/lib/generator/split.js +194 -54
  107. package/lib/generator/wizard-prompts-secondary.js +294 -0
  108. package/lib/generator/wizard-prompts.js +105 -106
  109. package/lib/generator/wizard-readme.js +88 -0
  110. package/lib/generator/wizard.js +155 -158
  111. package/lib/infrastructure/compose.js +11 -1
  112. package/lib/infrastructure/helpers.js +103 -20
  113. package/lib/infrastructure/index.js +98 -12
  114. package/lib/infrastructure/services.js +88 -22
  115. package/lib/schema/application-schema.json +32 -8
  116. package/lib/schema/external-datasource.schema.json +49 -26
  117. package/lib/schema/external-system.schema.json +509 -411
  118. package/lib/schema/wizard-config.schema.json +16 -0
  119. package/lib/utils/api.js +41 -13
  120. package/lib/utils/app-register-auth.js +25 -3
  121. package/lib/utils/auth-headers.js +8 -7
  122. package/lib/utils/cli-utils.js +20 -0
  123. package/lib/utils/compose-generator.js +77 -76
  124. package/lib/utils/compose-handlebars-helpers.js +54 -0
  125. package/lib/utils/compose-vector-helper.js +18 -0
  126. package/lib/utils/config-format-preference.js +51 -0
  127. package/lib/utils/config-format.js +36 -0
  128. package/lib/utils/config-paths.js +127 -2
  129. package/lib/utils/configuration-env-resolver.js +179 -0
  130. package/lib/utils/credential-display.js +83 -0
  131. package/lib/utils/credential-secrets-env.js +357 -0
  132. package/lib/utils/dataplane-pipeline-warning.js +28 -0
  133. package/lib/utils/deployment-validation-helpers.js +4 -4
  134. package/lib/utils/dev-ca-install.js +139 -0
  135. package/lib/utils/dev-cert-helper.js +122 -0
  136. package/lib/utils/device-code-helpers.js +224 -0
  137. package/lib/utils/device-code.js +37 -336
  138. package/lib/utils/docker-build.js +40 -8
  139. package/lib/utils/env-copy.js +103 -13
  140. package/lib/utils/env-map.js +35 -5
  141. package/lib/utils/env-template.js +6 -5
  142. package/lib/utils/error-formatters/http-status-errors.js +20 -2
  143. package/lib/utils/error-formatters/permission-errors.js +0 -1
  144. package/lib/utils/error-formatters/validation-errors.js +0 -1
  145. package/lib/utils/external-readme.js +56 -29
  146. package/lib/utils/external-system-display.js +59 -1
  147. package/lib/utils/external-system-test-helpers.js +21 -8
  148. package/lib/utils/external-system-validators.js +3 -0
  149. package/lib/utils/file-upload.js +20 -50
  150. package/lib/utils/help-builder.js +16 -2
  151. package/lib/utils/infra-status.js +80 -45
  152. package/lib/utils/local-secrets.js +7 -52
  153. package/lib/utils/mutagen-install.js +195 -0
  154. package/lib/utils/mutagen.js +146 -0
  155. package/lib/utils/paths.js +128 -37
  156. package/lib/utils/port-resolver.js +28 -16
  157. package/lib/utils/remote-dev-auth.js +38 -0
  158. package/lib/utils/remote-docker-env.js +43 -0
  159. package/lib/utils/remote-secrets-loader.js +60 -0
  160. package/lib/utils/secrets-canonical.js +93 -0
  161. package/lib/utils/secrets-generator.js +114 -6
  162. package/lib/utils/secrets-helpers.js +108 -114
  163. package/lib/utils/secrets-path.js +2 -2
  164. package/lib/utils/secrets-utils.js +52 -1
  165. package/lib/utils/secrets-validation.js +84 -0
  166. package/lib/utils/ssh-key-helper.js +116 -0
  167. package/lib/utils/test-log-writer.js +56 -0
  168. package/lib/utils/token-manager-messages.js +90 -0
  169. package/lib/utils/token-manager.js +29 -36
  170. package/lib/utils/variable-transformer.js +3 -3
  171. package/lib/validation/env-template-auth.js +157 -0
  172. package/lib/validation/env-template-kv.js +41 -0
  173. package/lib/validation/external-manifest-validator.js +25 -0
  174. package/lib/validation/external-system-auth-rules.js +86 -0
  175. package/lib/validation/validate-batch.js +149 -0
  176. package/lib/validation/validate-datasource-keys-api.js +33 -0
  177. package/lib/validation/validate-display.js +94 -16
  178. package/lib/validation/validate.js +25 -12
  179. package/lib/validation/validator.js +72 -9
  180. package/lib/validation/wizard-datasource-validation.js +50 -0
  181. package/package.json +8 -3
  182. package/scripts/install-local.js +34 -15
  183. package/templates/README.md +0 -1
  184. package/templates/applications/README.md.hbs +4 -4
  185. package/templates/applications/dataplane/application.yaml +6 -5
  186. package/templates/applications/dataplane/env.template +15 -10
  187. package/templates/applications/dataplane/rbac.yaml +2 -2
  188. package/templates/applications/keycloak/env.template +2 -0
  189. package/templates/applications/miso-controller/application.yaml +1 -0
  190. package/templates/applications/miso-controller/env.template +12 -10
  191. package/templates/external-system/README.md.hbs +65 -25
  192. package/templates/external-system/deploy.js.hbs +4 -2
  193. package/templates/external-system/external-datasource.yaml.hbs +217 -0
  194. package/templates/external-system/external-system.json.hbs +1 -18
  195. package/templates/infra/compose.yaml.hbs +6 -0
  196. package/templates/python/docker-compose.hbs +49 -23
  197. package/templates/typescript/docker-compose.hbs +48 -22
  198. package/integration/hubspot/application.yaml +0 -37
@@ -8,7 +8,7 @@
8
8
  * @author AI Fabrix Team
9
9
  * @version 2.0.0
10
10
  */
11
-
11
+ /* eslint-disable max-lines -- Central module; env-only resolve (plan 75) added required options; extract to env-merge would touch multiple callers. */
12
12
  const fs = require('fs');
13
13
  const path = require('path');
14
14
  const logger = require('../utils/logger');
@@ -27,20 +27,27 @@ const {
27
27
  ensureNonEmptySecrets,
28
28
  validateSecrets
29
29
  } = require('../utils/secrets-helpers');
30
- const { processEnvVariables } = require('../utils/env-copy');
31
30
  const { buildEnvVarMap } = require('../utils/env-map');
32
31
  const { resolveServicePortsInEnvContent } = require('../utils/secrets-url');
33
- const { updatePortForDocker } = require('./secrets-docker-env');
32
+ const {
33
+ updatePortForDocker,
34
+ getBaseDockerEnv,
35
+ applyDockerEnvOverride,
36
+ getContainerPortFromDockerEnv
37
+ } = require('./secrets-docker-env');
38
+ const { getContainerPortFromPath } = require('../utils/port-resolver');
34
39
  const {
35
40
  generateMissingSecrets,
36
41
  createDefaultSecrets
37
42
  } = require('../utils/secrets-generator');
43
+ const secretsEnsure = require('./secrets-ensure');
38
44
  const {
39
45
  resolveSecretsPath,
40
46
  getActualSecretsPath
41
47
  } = require('../utils/secrets-path');
42
48
  const {
43
49
  loadUserSecrets,
50
+ loadPrimaryUserSecrets,
44
51
  loadDefaultSecrets
45
52
  } = require('../utils/secrets-utils');
46
53
  const { decryptSecret, isEncrypted } = require('../utils/secrets-encryption');
@@ -162,12 +169,18 @@ function mergeUserWithConfigFile(userSecrets, resolvedConfigPath) {
162
169
  }
163
170
 
164
171
  /**
165
- * Loads config secrets path, merges with user secrets (user overrides). Used by loadSecrets cascade.
172
+ * Loads config secrets path, merges with user secrets (user/master wins, public fills missing).
173
+ * User/master = primary home (AIFABRIX_HOME or ~/.aifabrix) secrets.local.yaml.
174
+ * Public = aifabrix-secrets path from config. Used by loadSecrets cascade.
175
+ * When aifabrix-secrets is an http(s) URL, fetches shared secrets from API (never persisted to disk).
176
+ *
166
177
  * @async
167
178
  * @returns {Promise<Object|null>} Merged secrets object or null
168
179
  */
169
180
  async function loadMergedConfigAndUserSecrets() {
170
- const userSecrets = loadUserSecrets();
181
+ const { loadRemoteSharedSecrets, mergeUserWithRemoteSecrets } = require('../utils/remote-secrets-loader');
182
+ const { isRemoteSecretsUrl } = require('../utils/remote-dev-auth');
183
+ const userSecrets = loadPrimaryUserSecrets();
171
184
  const hasKeys = (obj) => obj && Object.keys(obj).length > 0;
172
185
  const userOrNull = () => (hasKeys(userSecrets) ? userSecrets : null);
173
186
  try {
@@ -175,6 +188,11 @@ async function loadMergedConfigAndUserSecrets() {
175
188
  if (!configSecretsPath) {
176
189
  return userOrNull();
177
190
  }
191
+ if (isRemoteSecretsUrl(configSecretsPath)) {
192
+ const remoteSecrets = await loadRemoteSharedSecrets();
193
+ const merged = mergeUserWithRemoteSecrets(userSecrets, remoteSecrets);
194
+ return hasKeys(merged) ? merged : userOrNull();
195
+ }
178
196
  const resolvedConfigPath = path.isAbsolute(configSecretsPath)
179
197
  ? configSecretsPath
180
198
  : path.resolve(process.cwd(), configSecretsPath);
@@ -188,6 +206,38 @@ async function loadMergedConfigAndUserSecrets() {
188
206
  }
189
207
  }
190
208
 
209
+ /**
210
+ * Loads merged secrets using config/user cascade, builder file merge, and default fallback.
211
+ * @async
212
+ * @returns {Promise<Object>} Merged secrets object (not decrypted)
213
+ */
214
+ async function loadSecretsWithFallbacks() {
215
+ let merged = await loadMergedConfigAndUserSecrets();
216
+ if (!merged || Object.keys(merged).length === 0) {
217
+ merged = loadPrimaryUserSecrets();
218
+ if (Object.keys(merged).length === 0) {
219
+ merged = loadUserSecrets();
220
+ }
221
+ merged = await applyCanonicalSecretsOverride(merged);
222
+ }
223
+ try {
224
+ const projectRoot = pathsUtil.getProjectRoot();
225
+ if (projectRoot) {
226
+ const builderPath = path.join(projectRoot, 'builder', 'secrets.local.yaml');
227
+ if (fs.existsSync(builderPath)) {
228
+ const builderSecrets = mergeUserWithConfigFile(merged || {}, builderPath);
229
+ if (builderSecrets) merged = builderSecrets;
230
+ }
231
+ }
232
+ } catch {
233
+ // Ignore (e.g. no project root or read error)
234
+ }
235
+ if (Object.keys(merged).length === 0) {
236
+ merged = loadDefaultSecrets();
237
+ }
238
+ return merged;
239
+ }
240
+
191
241
  async function loadSecrets(secretsPath, _appName) {
192
242
  if (secretsPath) {
193
243
  const resolvedPath = resolveSecretsPath(secretsPath);
@@ -200,15 +250,7 @@ async function loadSecrets(secretsPath, _appName) {
200
250
  }
201
251
  return await decryptSecretsObject(explicitSecrets);
202
252
  }
203
-
204
- let mergedSecrets = await loadMergedConfigAndUserSecrets();
205
- if (!mergedSecrets || Object.keys(mergedSecrets).length === 0) {
206
- mergedSecrets = loadUserSecrets();
207
- mergedSecrets = await applyCanonicalSecretsOverride(mergedSecrets);
208
- }
209
- if (Object.keys(mergedSecrets).length === 0) {
210
- mergedSecrets = loadDefaultSecrets();
211
- }
253
+ const mergedSecrets = await loadSecretsWithFallbacks();
212
254
  ensureNonEmptySecrets(mergedSecrets);
213
255
  return await decryptSecretsObject(mergedSecrets);
214
256
  }
@@ -259,44 +301,57 @@ async function resolveKvReferences(envTemplate, secrets, environment = 'local',
259
301
  return replaceKvInContent(resolved, secrets, envVars);
260
302
  }
261
303
 
262
- /** Applies environment-specific transformations to resolved content. */
304
+ /** Docker env transformations: ports, infra endpoints, PORT. */
305
+ async function applyDockerTransformations(resolved, variablesPath) {
306
+ resolved = await resolveServicePortsInEnvContent(resolved, 'docker');
307
+ resolved = await rewriteInfraEndpoints(resolved, 'docker');
308
+ const { getEnvHosts, getServiceHost, getServicePort, getLocalhostOverride } = require('../utils/env-endpoints');
309
+ const hosts = await getEnvHosts('docker');
310
+ const localhostOverride = getLocalhostOverride('docker');
311
+ const redisHost = getServiceHost(hosts.REDIS_HOST, 'docker', 'redis', localhostOverride);
312
+ const redisPort = await getServicePort('REDIS_PORT', 'redis', hosts, 'docker', null);
313
+ const dbHost = getServiceHost(hosts.DB_HOST, 'docker', 'postgres', localhostOverride);
314
+ const dbPort = await getServicePort('DB_PORT', 'postgres', hosts, 'docker', null);
315
+ let dockerEnv = await getBaseDockerEnv();
316
+ dockerEnv = applyDockerEnvOverride(dockerEnv);
317
+ const containerPort = getContainerPortFromPath(variablesPath) ?? getContainerPortFromDockerEnv(dockerEnv) ?? 3000;
318
+ const envVars = await buildEnvVarMap('docker', null, null, { appPort: containerPort });
319
+ envVars.REDIS_HOST = redisHost;
320
+ envVars.REDIS_PORT = String(redisPort);
321
+ envVars.DB_HOST = dbHost;
322
+ envVars.DB_PORT = String(dbPort);
323
+ envVars.PORT = String(containerPort);
324
+ resolved = interpolateEnvVars(resolved, envVars);
325
+ return updatePortForDocker(resolved, variablesPath);
326
+ }
327
+ /** Environment-specific transformations to resolved content. */
263
328
  async function applyEnvironmentTransformations(resolved, environment, variablesPath) {
264
- if (environment === 'docker') {
265
- resolved = await resolveServicePortsInEnvContent(resolved, environment);
266
- resolved = await rewriteInfraEndpoints(resolved, 'docker');
267
- const { getEnvHosts, getServiceHost, getServicePort, getLocalhostOverride } = require('../utils/env-endpoints');
268
- const hosts = await getEnvHosts('docker');
269
- const localhostOverride = getLocalhostOverride('docker');
270
- const redisHost = getServiceHost(hosts.REDIS_HOST, 'docker', 'redis', localhostOverride);
271
- const redisPort = await getServicePort('REDIS_PORT', 'redis', hosts, 'docker', null);
272
- const dbHost = getServiceHost(hosts.DB_HOST, 'docker', 'postgres', localhostOverride);
273
- const dbPort = await getServicePort('DB_PORT', 'postgres', hosts, 'docker', null);
274
- const envVars = await buildEnvVarMap('docker');
275
- envVars.REDIS_HOST = redisHost;
276
- envVars.REDIS_PORT = String(redisPort);
277
- envVars.DB_HOST = dbHost;
278
- envVars.DB_PORT = String(dbPort);
279
- resolved = interpolateEnvVars(resolved, envVars);
280
- resolved = await updatePortForDocker(resolved, variablesPath);
281
- } else if (environment === 'local') {
282
- // adjustLocalEnvPortsInContent handles both PORT and infra endpoints
283
- resolved = await adjustLocalEnvPortsInContent(resolved, variablesPath);
284
- }
329
+ if (environment === 'docker') return applyDockerTransformations(resolved, variablesPath);
330
+ if (environment === 'local') return adjustLocalEnvPortsInContent(resolved, variablesPath);
285
331
  return resolved;
286
332
  }
287
333
 
288
- /** Generates .env file content from template and secrets (without writing to disk). */
289
- async function generateEnvContent(appName, secretsPath, environment = 'local', force = false) {
290
- const builderPath = pathsUtil.getBuilderPath(appName);
291
- const templatePath = path.join(builderPath, 'env.template');
292
- const variablesPath = resolveApplicationConfigPath(builderPath);
334
+ /**
335
+ * Generate .env content from template and secrets (no disk write).
336
+ * When options.envOnly is true, variablesPath is null (no application config).
337
+ *
338
+ * @param {string} appName - Application name
339
+ * @param {string} [secretsPath] - Path to secrets file (optional)
340
+ * @param {string} [environment='local'] - Environment context
341
+ * @param {boolean} [force=false] - Generate missing secret keys
342
+ * @param {Object} [options] - Optional: appPath, envOnly (env-only mode uses only env.template)
343
+ * @returns {Promise<string>} Resolved env content
344
+ */
345
+ async function generateEnvContent(appName, secretsPath, environment = 'local', force = false, options = {}) {
346
+ const appPath = (options && options.appPath) || pathsUtil.getBuilderPath(appName);
347
+ const templatePath = path.join(appPath, 'env.template');
348
+ const variablesPath = (options && options.envOnly) ? null : resolveApplicationConfigPath(appPath);
293
349
  const template = loadEnvTemplate(templatePath);
294
350
  const secretsPaths = await getActualSecretsPath(secretsPath, appName);
295
351
  if (force) {
296
- const secretsFileForGeneration = secretsPath ? resolveSecretsPath(secretsPath) : secretsPaths.userPath;
297
- await generateMissingSecrets(template, secretsFileForGeneration);
352
+ const preferredPath = secretsPath ? resolveSecretsPath(secretsPath) : secretsPaths.userPath;
353
+ await secretsEnsure.ensureSecretsFromEnvTemplate(templatePath, { preferredFilePath: preferredPath });
298
354
  }
299
-
300
355
  const secrets = await loadSecrets(secretsPath, appName);
301
356
  let resolved = await resolveKvReferences(template, secrets, environment, secretsPaths, appName);
302
357
  resolved = await applyEnvironmentTransformations(resolved, environment, variablesPath);
@@ -371,73 +426,104 @@ function mergeEnvContentPreservingExisting(newContent, existingMap) {
371
426
  }
372
427
 
373
428
  /**
374
- * Generates and writes .env file. Newly resolved values (from template + secrets) win over
375
- * existing .env so that project secrets (e.g. from config aifabrix-secrets) take effect on
376
- * re-run. Extra variables only in existing .env are kept.
429
+ * Merges a key-value map into existing .env file content, preserving comments and blank lines.
430
+ * For each KEY=value line in existing content, replaces value with newMap[KEY] when the key exists
431
+ * in newMap. Appends any keys from newMap that did not appear in the file.
377
432
  *
433
+ * @function mergeEnvMapIntoContent
434
+ * @param {string} existingContent - Full existing .env file content
435
+ * @param {Object.<string, string>} newMap - New key-to-value map (e.g. from resolved or run env)
436
+ * @returns {string} Merged content with comments preserved
437
+ */
438
+ function mergeEnvMapIntoContent(existingContent, newMap) {
439
+ if (!newMap || Object.keys(newMap).length === 0) {
440
+ return typeof existingContent === 'string' ? existingContent : '';
441
+ }
442
+ const lines = (existingContent || '').split(/\r?\n/);
443
+ const seen = new Set();
444
+ const out = [];
445
+ for (const line of lines) {
446
+ const trimmed = line.trim();
447
+ if (!trimmed || trimmed.startsWith('#')) {
448
+ out.push(line);
449
+ continue;
450
+ }
451
+ const eq = trimmed.indexOf('=');
452
+ if (eq > 0) {
453
+ const key = trimmed.substring(0, eq).trim();
454
+ seen.add(key);
455
+ out.push(Object.prototype.hasOwnProperty.call(newMap, key) ? `${key}=${newMap[key]}` : line);
456
+ continue;
457
+ }
458
+ out.push(line);
459
+ }
460
+ for (const key of Object.keys(newMap)) {
461
+ if (!seen.has(key)) out.push(`${key}=${newMap[key]}`);
462
+ }
463
+ return out.join('\n');
464
+ }
465
+
466
+ /**
467
+ * Resolves content to write for .env: merges with existing file when present.
468
+ * @param {string} resolved - Newly generated content
469
+ * @param {string} pathToPreserve - Path to existing .env to merge from (or null)
470
+ * @returns {string} Content to write
471
+ */
472
+ function resolveEnvContentToWrite(resolved, pathToPreserve) {
473
+ if (!pathToPreserve || !fs.existsSync(pathToPreserve)) return resolved;
474
+ const existingContent = fs.readFileSync(pathToPreserve, 'utf8');
475
+ const existingMap = parseEnvContentToMap(existingContent);
476
+ return mergeEnvContentPreservingExisting(resolved, existingMap);
477
+ }
478
+
479
+ /**
480
+ * Generates and writes .env file. Newly resolved values win over existing .env; extra vars in existing .env are kept.
481
+ * When options.envOnly is true, only env.template is used; .env is written to options.appPath.
378
482
  * @async
379
483
  * @function generateEnvFile
380
484
  * @param {string} appName - Name of the application
381
485
  * @param {string} [secretsPath] - Path to secrets file (optional)
382
486
  * @param {string} [environment='local'] - Environment context ('local' or 'docker')
383
487
  * @param {boolean} [force=false] - Generate missing secret keys in secrets file
384
- * @param {boolean} [skipOutputPath=false] - Skip copying to envOutputPath
385
- * @param {string} [preserveFromPath=null] - Path to existing .env to preserve values from (defaults to builder .env)
488
+ * @param {Object} [options] - Optional: appPath, envOnly, skipOutputPath, preserveFromPath
386
489
  * @returns {Promise<string>} Path to generated .env file
387
- * @throws {Error} If generation fails
388
- *
389
- * @example
390
- * const envPath = await generateEnvFile('myapp', undefined, 'docker');
391
- * // When builder/myapp/.env already exists, existing values are preserved
392
490
  */
393
- async function generateEnvFile(appName, secretsPath, environment = 'local', force = false, skipOutputPath = false, preserveFromPath = null) {
394
- const builderPath = pathsUtil.getBuilderPath(appName);
395
- const variablesPath = resolveApplicationConfigPath(builderPath);
396
- const envPath = path.join(builderPath, '.env');
397
-
398
- const resolved = await generateEnvContent(appName, secretsPath, environment, force);
399
-
400
- let toWrite = resolved;
401
- const pathToPreserve = preserveFromPath || envPath;
402
- if (pathToPreserve && fs.existsSync(pathToPreserve)) {
403
- const existingContent = fs.readFileSync(pathToPreserve, 'utf8');
404
- const existingMap = parseEnvContentToMap(existingContent);
405
- toWrite = mergeEnvContentPreservingExisting(resolved, existingMap);
491
+ async function generateEnvFile(appName, secretsPath, environment = 'local', force = false, options = {}) {
492
+ const opts = options && typeof options === 'object' ? options : {};
493
+ const appPath = opts.appPath || pathsUtil.getBuilderPath(appName);
494
+ const envOnly = !!opts.envOnly;
495
+ const variablesPath = envOnly ? null : resolveApplicationConfigPath(appPath);
496
+ const envPath = path.join(appPath, '.env');
497
+
498
+ if (envOnly) {
499
+ const templatePath = path.join(appPath, 'env.template');
500
+ if (!fs.existsSync(templatePath)) {
501
+ throw new Error(`env.template not found at ${templatePath}. Resolve requires env.template in the app directory.`);
502
+ }
406
503
  }
407
504
 
505
+ const resolved = await generateEnvContent(appName, secretsPath, environment, force, { appPath, envOnly });
506
+ const preservePath = opts.preserveFromPath !== undefined && opts.preserveFromPath !== null ? opts.preserveFromPath : null;
507
+ const pathToPreserve = preservePath !== null ? preservePath : envPath;
508
+ const toWrite = resolveEnvContentToWrite(resolved, pathToPreserve);
408
509
  fs.writeFileSync(envPath, toWrite, { mode: 0o600 });
409
510
 
410
- // Process and copy to envOutputPath if configured (uses localPort for copied file)
411
- if (!skipOutputPath) {
511
+ if (!opts.skipOutputPath) {
512
+ const { processEnvVariables } = require('../utils/env-copy');
412
513
  await processEnvVariables(envPath, variablesPath, appName, secretsPath);
413
514
  }
414
515
 
415
516
  return envPath;
416
517
  }
417
518
 
418
- /**
419
- * Generates admin secrets for infrastructure
420
- * Creates ~/.aifabrix/admin-secrets.env with Postgres and Redis credentials
421
- *
422
- * @async
423
- * @function generateAdminSecretsEnv
424
- * @param {string} [secretsPath] - Path to secrets file (optional)
425
- * @returns {Promise<string>} Path to generated admin-secrets.env file
426
- * @throws {Error} If generation fails
427
- *
428
- * @example
429
- * const adminEnvPath = await generateAdminSecretsEnv('../../secrets.local.yaml');
430
- * // Returns: '~/.aifabrix/admin-secrets.env'
431
- */
519
+ /** Generates admin secrets for infrastructure (~/.aifabrix/admin-secrets.env). Uses admin123 when no postgres password. */
432
520
  async function generateAdminSecretsEnv(secretsPath) {
433
521
  let secrets;
434
522
 
435
523
  try {
436
524
  secrets = await loadSecrets(secretsPath);
437
525
  } catch (error) {
438
- // If secrets file doesn't exist, create default secrets
439
526
  const defaultSecretsPath = secretsPath || path.join(pathsUtil.getAifabrixHome(), 'secrets.yaml');
440
-
441
527
  if (!fs.existsSync(defaultSecretsPath)) {
442
528
  logger.log('Creating default secrets file...');
443
529
  await createDefaultSecrets(defaultSecretsPath);
@@ -446,15 +532,14 @@ async function generateAdminSecretsEnv(secretsPath) {
446
532
  throw error;
447
533
  }
448
534
  }
449
-
450
535
  const aifabrixDir = pathsUtil.getAifabrixHome();
451
536
  const adminEnvPath = path.join(aifabrixDir, 'admin-secrets.env');
452
-
453
537
  if (!fs.existsSync(aifabrixDir)) {
454
538
  fs.mkdirSync(aifabrixDir, { recursive: true, mode: 0o700 });
455
539
  }
456
540
 
457
- const postgresPassword = secrets['postgres-passwordKeyVault'] || '';
541
+ const raw = secrets['postgres-passwordKeyVault'];
542
+ const postgresPassword = (raw && String(raw).trim()) || 'admin123';
458
543
 
459
544
  const adminSecrets = `# Infrastructure Admin Credentials
460
545
  POSTGRES_PASSWORD=${postgresPassword}
@@ -477,5 +562,7 @@ module.exports = {
477
562
  generateAdminSecretsEnv,
478
563
  validateSecrets,
479
564
  createDefaultSecrets,
480
- getCanonicalSecretName
565
+ getCanonicalSecretName,
566
+ parseEnvContentToMap,
567
+ mergeEnvMapIntoContent
481
568
  };
@@ -16,6 +16,11 @@ const { getEnvironmentApplication } = require('../api/environments.api');
16
16
  const { publishDatasourceViaPipeline } = require('../api/pipeline.api');
17
17
  const { formatApiError } = require('../utils/api-error-handler');
18
18
  const logger = require('../utils/logger');
19
+ const { logDataplanePipelineWarning } = require('../utils/dataplane-pipeline-warning');
20
+ const {
21
+ buildResolvedEnvMapForIntegration,
22
+ resolveConfigurationValues
23
+ } = require('../utils/configuration-env-resolver');
19
24
  const { validateDatasourceFile } = require('./validate');
20
25
 
21
26
  /**
@@ -50,7 +55,7 @@ async function getDataplaneUrl(controllerUrl, appKey, environment, authConfig) {
50
55
  if (!dataplaneUrl) {
51
56
  const appType = application.configuration?.type || application.type;
52
57
  if (appType === 'external') {
53
- throw new Error('Dataplane URL not found for external system. Provide --dataplane <url>.');
58
+ throw new Error('Dataplane URL not found for external system in application configuration.');
54
59
  }
55
60
  throw new Error('Dataplane URL not found in application configuration');
56
61
  }
@@ -107,8 +112,6 @@ async function validateAndLoadDatasourceFile(filePath) {
107
112
  * @param {string} controllerUrl - Controller URL
108
113
  * @param {string} environment - Environment key
109
114
  * @param {string} appKey - Application key
110
- * @param {Object} [options] - Options
111
- * @param {string} [options.dataplane] - Dataplane URL override
112
115
  * @returns {Promise<Object>} Object with authConfig and dataplaneUrl
113
116
  */
114
117
  async function setupDeploymentAuth(controllerUrl, environment, appKey) {
@@ -154,6 +157,7 @@ async function setupDeploymentAuth(controllerUrl, environment, appKey) {
154
157
  async function publishDatasourceToDataplane(dataplaneUrl, systemKey, authConfig, datasourceConfig) {
155
158
  requireBearerForDataplanePipeline(authConfig);
156
159
  logger.log(chalk.blue('\nšŸš€ Publishing datasource to dataplane...'));
160
+ logDataplanePipelineWarning();
157
161
 
158
162
  const publishResponse = await publishDatasourceViaPipeline(dataplaneUrl, systemKey, authConfig, datasourceConfig);
159
163
 
@@ -228,6 +232,11 @@ async function deployDatasource(appKey, filePath, _options) {
228
232
  throw new Error('systemKey is required in datasource configuration');
229
233
  }
230
234
 
235
+ if (Array.isArray(datasourceConfig.configuration) && datasourceConfig.configuration.length > 0) {
236
+ const { envMap, secrets } = await buildResolvedEnvMapForIntegration(systemKey);
237
+ resolveConfigurationValues(datasourceConfig.configuration, envMap, secrets, systemKey);
238
+ }
239
+
231
240
  // Setup authentication and get dataplane URL
232
241
  const { authConfig, dataplaneUrl } = await setupDeploymentAuth(controllerUrl, environment, appKey);
233
242
 
@@ -0,0 +1,91 @@
1
+ /**
2
+ * Field Reference Validator for External Datasource
3
+ *
4
+ * Validates that field names used in indexing (embedding, uniqueKey),
5
+ * validation.repeatingValues[].field, and quality.rejectIf[].field exist in
6
+ * fieldMappings.attributes. Aligns with dataplane invalid_reference semantics.
7
+ *
8
+ * @fileoverview Offline field reference validation for AI Fabrix Builder
9
+ * @author AI Fabrix Team
10
+ * @version 2.0.0
11
+ */
12
+
13
+ /**
14
+ * Validates that all field references in indexing, validation, and quality
15
+ * exist in fieldMappings.attributes. When fieldMappings.attributes is missing
16
+ * or empty, returns no errors (skip check, matching dataplane behavior).
17
+ *
18
+ * @function validateFieldReferences
19
+ * @param {Object} parsed - Parsed datasource object (after JSON parse)
20
+ * @returns {string[]} Array of error messages; empty if no invalid references
21
+ *
22
+ * @example
23
+ * const errors = validateFieldReferences(parsed);
24
+ * if (errors.length > 0) {
25
+ * errors.forEach(e => console.error(e));
26
+ * }
27
+ */
28
+ function validateFieldReferences(parsed) {
29
+ const errors = [];
30
+ const normalizedAttributes = Object.keys(
31
+ parsed?.fieldMappings?.attributes ?? {}
32
+ );
33
+
34
+ if (normalizedAttributes.length === 0) {
35
+ return [];
36
+ }
37
+
38
+ // indexing.embedding: array of field names
39
+ const embedding = parsed?.indexing?.embedding;
40
+ if (Array.isArray(embedding)) {
41
+ embedding.forEach((field, i) => {
42
+ if (typeof field === 'string' && !normalizedAttributes.includes(field)) {
43
+ errors.push(
44
+ `indexing.embedding[${i}]: field '${field}' does not exist in fieldMappings.attributes`
45
+ );
46
+ }
47
+ });
48
+ }
49
+
50
+ // indexing.uniqueKey: single field name
51
+ const uniqueKey = parsed?.indexing?.uniqueKey;
52
+ if (typeof uniqueKey === 'string' && uniqueKey !== '') {
53
+ if (!normalizedAttributes.includes(uniqueKey)) {
54
+ errors.push(
55
+ `indexing.uniqueKey: field '${uniqueKey}' does not exist in fieldMappings.attributes`
56
+ );
57
+ }
58
+ }
59
+
60
+ // validation.repeatingValues[].field
61
+ const repeatingValues = parsed?.validation?.repeatingValues;
62
+ if (Array.isArray(repeatingValues)) {
63
+ repeatingValues.forEach((rule, index) => {
64
+ const field = rule?.field;
65
+ if (typeof field === 'string' && !normalizedAttributes.includes(field)) {
66
+ errors.push(
67
+ `validation.repeatingValues[${index}].field: field '${field}' does not exist in fieldMappings.attributes`
68
+ );
69
+ }
70
+ });
71
+ }
72
+
73
+ // quality.rejectIf[].field
74
+ const rejectIf = parsed?.quality?.rejectIf;
75
+ if (Array.isArray(rejectIf)) {
76
+ rejectIf.forEach((rule, index) => {
77
+ const field = rule?.field;
78
+ if (typeof field === 'string' && !normalizedAttributes.includes(field)) {
79
+ errors.push(
80
+ `quality.rejectIf[${index}].field: field '${field}' does not exist in fieldMappings.attributes`
81
+ );
82
+ }
83
+ });
84
+ }
85
+
86
+ return errors;
87
+ }
88
+
89
+ module.exports = {
90
+ validateFieldReferences
91
+ };