@aifabrix/builder 2.40.2 → 2.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/.cursor/rules/docs-rules.mdc +30 -0
  2. package/README.md +7 -5
  3. package/integration/hubspot/README.md +8 -4
  4. package/integration/hubspot/application.json +54 -0
  5. package/integration/hubspot/create-hubspot.js +9 -136
  6. package/integration/hubspot/env.template +3 -4
  7. package/integration/hubspot/hubspot-datasource-company.json +343 -5
  8. package/integration/hubspot/hubspot-datasource-contact.json +413 -5
  9. package/integration/hubspot/hubspot-datasource-deal.json +341 -4
  10. package/integration/hubspot/hubspot-datasource-users.json +116 -0
  11. package/integration/hubspot/hubspot-deploy.json +1250 -108
  12. package/integration/hubspot/hubspot-system.json +15 -32
  13. package/integration/hubspot/test-dataplane-down-tests.js +17 -16
  14. package/integration/hubspot/test-dataplane-down.js +2 -2
  15. package/integration/hubspot/test.js +1 -1
  16. package/jest.config.manual.js +2 -1
  17. package/lib/api/credential.api.js +40 -0
  18. package/lib/api/dev.api.js +423 -0
  19. package/lib/api/external-test.api.js +111 -0
  20. package/lib/api/index.js +42 -19
  21. package/lib/api/pipeline.api.js +66 -120
  22. package/lib/api/types/credential.types.js +23 -0
  23. package/lib/api/types/dev.types.js +140 -0
  24. package/lib/api/types/pipeline.types.js +37 -0
  25. package/lib/api/wizard-platform.api.js +61 -0
  26. package/lib/api/wizard.api.js +34 -1
  27. package/lib/app/config.js +44 -11
  28. package/lib/app/down.js +2 -1
  29. package/lib/app/index.js +12 -1
  30. package/lib/app/prompts.js +44 -29
  31. package/lib/app/push.js +36 -12
  32. package/lib/app/readme.js +9 -6
  33. package/lib/app/run-env-compose.js +264 -0
  34. package/lib/app/run-helpers.js +121 -118
  35. package/lib/app/run.js +148 -28
  36. package/lib/app/show-display.js +1 -1
  37. package/lib/app/show.js +5 -2
  38. package/lib/build/index.js +11 -3
  39. package/lib/cli/setup-app.js +172 -15
  40. package/lib/cli/setup-credential-deployment.js +31 -6
  41. package/lib/cli/setup-dev.js +206 -16
  42. package/lib/cli/setup-environment.js +16 -6
  43. package/lib/cli/setup-external-system.js +89 -24
  44. package/lib/cli/setup-infra.js +82 -15
  45. package/lib/cli/setup-secrets.js +52 -5
  46. package/lib/cli/setup-utility.js +129 -24
  47. package/lib/commands/app-install.js +172 -0
  48. package/lib/commands/app-shell.js +75 -0
  49. package/lib/commands/app-test.js +282 -0
  50. package/lib/commands/app.js +1 -1
  51. package/lib/commands/credential-env.js +162 -0
  52. package/lib/commands/credential-list.js +17 -22
  53. package/lib/commands/credential-push.js +96 -0
  54. package/lib/commands/datasource.js +77 -6
  55. package/lib/commands/dev-cli-handlers.js +141 -0
  56. package/lib/commands/dev-down.js +114 -0
  57. package/lib/commands/dev-init.js +347 -0
  58. package/lib/commands/repair-auth-config.js +99 -0
  59. package/lib/commands/repair-datasource-keys.js +208 -0
  60. package/lib/commands/repair-datasource.js +235 -0
  61. package/lib/commands/repair-env-template.js +348 -0
  62. package/lib/commands/repair-internal.js +85 -0
  63. package/lib/commands/repair-rbac.js +158 -0
  64. package/lib/commands/repair.js +507 -0
  65. package/lib/commands/secrets-list.js +118 -0
  66. package/lib/commands/secrets-remove.js +97 -0
  67. package/lib/commands/secrets-set.js +30 -17
  68. package/lib/commands/secrets-validate.js +50 -0
  69. package/lib/commands/test-e2e-external.js +165 -0
  70. package/lib/commands/up-dataplane.js +2 -2
  71. package/lib/commands/up-miso.js +0 -25
  72. package/lib/commands/upload.js +96 -40
  73. package/lib/commands/wizard-core-helpers.js +226 -4
  74. package/lib/commands/wizard-core.js +67 -29
  75. package/lib/commands/wizard-dataplane.js +1 -1
  76. package/lib/commands/wizard-entity-selection.js +43 -0
  77. package/lib/commands/wizard-headless.js +44 -5
  78. package/lib/commands/wizard-helpers.js +7 -3
  79. package/lib/commands/wizard.js +86 -64
  80. package/lib/core/admin-secrets.js +96 -0
  81. package/lib/core/config.js +7 -1
  82. package/lib/core/secrets-ensure.js +378 -0
  83. package/lib/core/secrets-env-write.js +157 -0
  84. package/lib/core/secrets.js +176 -89
  85. package/lib/datasource/deploy.js +12 -3
  86. package/lib/datasource/field-reference-validator.js +91 -0
  87. package/lib/datasource/test-e2e.js +219 -0
  88. package/lib/datasource/test-integration.js +154 -0
  89. package/lib/datasource/validate.js +21 -3
  90. package/lib/deployment/deployer.js +7 -5
  91. package/lib/deployment/environment-config.js +137 -0
  92. package/lib/deployment/environment.js +21 -98
  93. package/lib/deployment/push.js +32 -2
  94. package/lib/external-system/download.js +188 -203
  95. package/lib/external-system/generator.js +204 -56
  96. package/lib/external-system/test-auth.js +7 -3
  97. package/lib/external-system/test-execution.js +2 -1
  98. package/lib/external-system/test-system-level.js +73 -0
  99. package/lib/external-system/test.js +56 -19
  100. package/lib/generator/external-controller-manifest.js +29 -2
  101. package/lib/generator/external-schema-utils.js +1 -1
  102. package/lib/generator/external.js +10 -3
  103. package/lib/generator/index.js +177 -25
  104. package/lib/generator/split-readme.js +1 -0
  105. package/lib/generator/split-variables.js +7 -1
  106. package/lib/generator/split.js +194 -54
  107. package/lib/generator/wizard-prompts-secondary.js +294 -0
  108. package/lib/generator/wizard-prompts.js +105 -106
  109. package/lib/generator/wizard-readme.js +88 -0
  110. package/lib/generator/wizard.js +155 -158
  111. package/lib/infrastructure/compose.js +11 -1
  112. package/lib/infrastructure/helpers.js +103 -20
  113. package/lib/infrastructure/index.js +98 -12
  114. package/lib/infrastructure/services.js +88 -22
  115. package/lib/schema/application-schema.json +32 -8
  116. package/lib/schema/external-datasource.schema.json +49 -26
  117. package/lib/schema/external-system.schema.json +509 -411
  118. package/lib/schema/wizard-config.schema.json +16 -0
  119. package/lib/utils/api.js +41 -13
  120. package/lib/utils/app-register-auth.js +25 -3
  121. package/lib/utils/auth-headers.js +8 -7
  122. package/lib/utils/cli-utils.js +20 -0
  123. package/lib/utils/compose-generator.js +77 -76
  124. package/lib/utils/compose-handlebars-helpers.js +54 -0
  125. package/lib/utils/compose-vector-helper.js +18 -0
  126. package/lib/utils/config-format-preference.js +51 -0
  127. package/lib/utils/config-format.js +36 -0
  128. package/lib/utils/config-paths.js +127 -2
  129. package/lib/utils/configuration-env-resolver.js +179 -0
  130. package/lib/utils/credential-display.js +83 -0
  131. package/lib/utils/credential-secrets-env.js +357 -0
  132. package/lib/utils/dataplane-pipeline-warning.js +28 -0
  133. package/lib/utils/deployment-validation-helpers.js +4 -4
  134. package/lib/utils/dev-ca-install.js +139 -0
  135. package/lib/utils/dev-cert-helper.js +122 -0
  136. package/lib/utils/device-code-helpers.js +224 -0
  137. package/lib/utils/device-code.js +37 -336
  138. package/lib/utils/docker-build.js +40 -8
  139. package/lib/utils/env-copy.js +103 -13
  140. package/lib/utils/env-map.js +35 -5
  141. package/lib/utils/env-template.js +6 -5
  142. package/lib/utils/error-formatters/http-status-errors.js +20 -2
  143. package/lib/utils/error-formatters/permission-errors.js +0 -1
  144. package/lib/utils/error-formatters/validation-errors.js +0 -1
  145. package/lib/utils/external-readme.js +56 -29
  146. package/lib/utils/external-system-display.js +59 -1
  147. package/lib/utils/external-system-test-helpers.js +21 -8
  148. package/lib/utils/external-system-validators.js +3 -0
  149. package/lib/utils/file-upload.js +20 -50
  150. package/lib/utils/help-builder.js +16 -2
  151. package/lib/utils/infra-status.js +80 -45
  152. package/lib/utils/local-secrets.js +7 -52
  153. package/lib/utils/mutagen-install.js +195 -0
  154. package/lib/utils/mutagen.js +146 -0
  155. package/lib/utils/paths.js +128 -37
  156. package/lib/utils/port-resolver.js +28 -16
  157. package/lib/utils/remote-dev-auth.js +38 -0
  158. package/lib/utils/remote-docker-env.js +43 -0
  159. package/lib/utils/remote-secrets-loader.js +60 -0
  160. package/lib/utils/secrets-canonical.js +93 -0
  161. package/lib/utils/secrets-generator.js +114 -6
  162. package/lib/utils/secrets-helpers.js +108 -114
  163. package/lib/utils/secrets-path.js +2 -2
  164. package/lib/utils/secrets-utils.js +52 -1
  165. package/lib/utils/secrets-validation.js +84 -0
  166. package/lib/utils/ssh-key-helper.js +116 -0
  167. package/lib/utils/test-log-writer.js +56 -0
  168. package/lib/utils/token-manager-messages.js +90 -0
  169. package/lib/utils/token-manager.js +29 -36
  170. package/lib/utils/variable-transformer.js +3 -3
  171. package/lib/validation/env-template-auth.js +157 -0
  172. package/lib/validation/env-template-kv.js +41 -0
  173. package/lib/validation/external-manifest-validator.js +25 -0
  174. package/lib/validation/external-system-auth-rules.js +86 -0
  175. package/lib/validation/validate-batch.js +149 -0
  176. package/lib/validation/validate-datasource-keys-api.js +33 -0
  177. package/lib/validation/validate-display.js +94 -16
  178. package/lib/validation/validate.js +25 -12
  179. package/lib/validation/validator.js +72 -9
  180. package/lib/validation/wizard-datasource-validation.js +50 -0
  181. package/package.json +8 -3
  182. package/scripts/install-local.js +34 -15
  183. package/templates/README.md +0 -1
  184. package/templates/applications/README.md.hbs +4 -4
  185. package/templates/applications/dataplane/application.yaml +6 -5
  186. package/templates/applications/dataplane/env.template +15 -10
  187. package/templates/applications/dataplane/rbac.yaml +2 -2
  188. package/templates/applications/keycloak/env.template +2 -0
  189. package/templates/applications/miso-controller/application.yaml +1 -0
  190. package/templates/applications/miso-controller/env.template +12 -10
  191. package/templates/external-system/README.md.hbs +65 -25
  192. package/templates/external-system/deploy.js.hbs +4 -2
  193. package/templates/external-system/external-datasource.yaml.hbs +217 -0
  194. package/templates/external-system/external-system.json.hbs +1 -18
  195. package/templates/infra/compose.yaml.hbs +6 -0
  196. package/templates/python/docker-compose.hbs +49 -23
  197. package/templates/typescript/docker-compose.hbs +48 -22
  198. package/integration/hubspot/application.yaml +0 -37
@@ -18,6 +18,7 @@ const { loadEnvConfig } = require('./env-config-loader');
18
18
  const { updateContainerPortInEnvFile } = require('./env-ports');
19
19
  const { buildEnvVarMap } = require('./env-map');
20
20
  const { getLocalPortFromPath } = require('./port-resolver');
21
+ const { readYamlAtPath, applyCanonicalSecretsOverride } = require('./secrets-canonical');
21
22
 
22
23
  /**
23
24
  * Interpolate ${VAR} occurrences with values from envVars map
@@ -42,25 +43,84 @@ function isCommentOrEmptyLine(line) {
42
43
  return t === '' || t.startsWith('#');
43
44
  }
44
45
 
46
+ /** Regex for kv:// path (allows slashes, e.g. kv://hubspot/clientId) */
47
+ const KV_REF_PATTERN = /kv:\/\/([a-zA-Z0-9_\-/]+)/g;
48
+
49
+ /**
50
+ * Find object key that matches part case-insensitively.
51
+ * @param {Object} obj - Object to search
52
+ * @param {string} part - Key to match (e.g. 'clientid')
53
+ * @returns {string|undefined} Actual key in obj or undefined
54
+ */
55
+ function findKeyCaseInsensitive(obj, part) {
56
+ if (!obj || typeof obj !== 'object' || part === null || part === undefined) return undefined;
57
+ const lower = String(part).toLowerCase();
58
+ for (const key of Object.keys(obj)) {
59
+ if (key.toLowerCase() === lower) return key;
60
+ }
61
+ return undefined;
62
+ }
63
+
45
64
  /**
46
- * Collect missing kv:// secrets referenced in content (skips commented and empty lines)
65
+ * Resolve value by walking path parts (nested keys).
66
+ * @param {Object} secrets - Root secrets object
67
+ * @param {string[]} parts - Path parts (e.g. ['hubspot', 'clientId'])
68
+ * @returns {*} Value or undefined
69
+ */
70
+ function getValueByNestedPath(secrets, parts) {
71
+ let value = secrets;
72
+ for (const part of parts) {
73
+ if (!value || typeof value !== 'object') return undefined;
74
+ const key = part in value ? part : findKeyCaseInsensitive(value, part);
75
+ value = key !== undefined ? value[key] : undefined;
76
+ if (value === undefined) return undefined;
77
+ }
78
+ return value;
79
+ }
80
+
81
+ /**
82
+ * Get secret value by path. Supports flat key (hubspot/clientId), nested object (hubspot.clientId),
83
+ * and case-insensitive matching (clientid matches clientId). Path-style and hyphen-style are distinct:
84
+ * hubspot/clientid and hubspot-clientid are different keys.
85
+ * @param {Object} secrets - Secrets object (may be nested)
86
+ * @param {string} pathStr - Path after kv:// (e.g. 'hubspot/clientId' or 'hubspot/clientid')
87
+ * @returns {*} Value or undefined if not found
88
+ */
89
+ function getValueByPath(secrets, pathStr) {
90
+ if (!secrets || typeof secrets !== 'object' || !pathStr) {
91
+ return undefined;
92
+ }
93
+ const direct = secrets[pathStr];
94
+ if (direct !== undefined) return direct;
95
+ const flatKey = findKeyCaseInsensitive(secrets, pathStr);
96
+ if (flatKey !== undefined) return secrets[flatKey];
97
+ if (!pathStr.includes('/')) return undefined;
98
+ return getValueByNestedPath(secrets, pathStr.split('/'));
99
+ }
100
+
101
+ /**
102
+ * Collect missing kv:// secrets referenced in content (skips commented and empty lines).
103
+ * Supports path-style refs (e.g. kv://hubspot/clientId). Returns unique refs.
47
104
  * @function collectMissingSecrets
48
105
  * @param {string} content - Text content
49
- * @param {Object} secrets - Available secrets
50
- * @returns {string[]} Array of missing kv://<key> references
106
+ * @param {Object} secrets - Available secrets (flat or nested)
107
+ * @returns {string[]} Array of missing kv://<path> references (unique)
51
108
  */
52
109
  function collectMissingSecrets(content, secrets) {
53
- const kvPattern = /kv:\/\/([a-zA-Z0-9-_]+)/g;
110
+ const seen = new Set();
54
111
  const missing = [];
55
112
  const lines = content.split('\n');
56
113
  for (const line of lines) {
57
114
  if (isCommentOrEmptyLine(line)) continue;
58
115
  let match;
59
- kvPattern.lastIndex = 0;
60
- while ((match = kvPattern.exec(line)) !== null) {
61
- const secretKey = match[1];
62
- if (!(secretKey in secrets)) {
63
- missing.push(`kv://${secretKey}`);
116
+ KV_REF_PATTERN.lastIndex = 0;
117
+ while ((match = KV_REF_PATTERN.exec(line)) !== null) {
118
+ const pathStr = match[1];
119
+ if (seen.has(pathStr)) continue;
120
+ seen.add(pathStr);
121
+ const value = getValueByPath(secrets, pathStr);
122
+ if (value === undefined || value === null) {
123
+ missing.push(`kv://${pathStr}`);
64
124
  }
65
125
  }
66
126
  }
@@ -91,26 +151,26 @@ function formatMissingSecretsFileInfo(secretsFilePaths) {
91
151
  }
92
152
 
93
153
  /**
94
- * Replace kv:// references with actual values (skips commented and empty lines)
154
+ * Replace kv:// references with actual values (skips commented and empty lines).
155
+ * Supports path-style refs (e.g. kv://hubspot/clientId) and nested secrets.
95
156
  * @function replaceKvInContent
96
157
  * @param {string} content - Text content containing kv:// references
97
- * @param {Object} secrets - Secrets map
158
+ * @param {Object} secrets - Secrets map (flat or nested)
98
159
  * @param {Object} envVars - Environment variables map for nested interpolation
99
160
  * @returns {string} Content with kv:// references replaced
100
161
  */
101
162
  function replaceKvInContent(content, secrets, envVars) {
102
- const kvPattern = /kv:\/\/([a-zA-Z0-9-_]+)/g;
103
163
  const lines = content.split('\n');
104
164
  const result = lines.map(line => {
105
165
  if (isCommentOrEmptyLine(line)) return line;
106
- return line.replace(kvPattern, (match, secretKey) => {
107
- let value = secrets[secretKey];
166
+ return line.replace(KV_REF_PATTERN, (match, pathStr) => {
167
+ let value = getValueByPath(secrets, pathStr);
108
168
  if (typeof value === 'string') {
109
169
  value = value.replace(/\$\{([A-Z_]+)\}/g, (m, envVar) => {
110
170
  return envVars[envVar] || m;
111
171
  });
112
172
  }
113
- return value;
173
+ return value !== null && value !== undefined ? String(value) : match;
114
174
  });
115
175
  });
116
176
  return result.join('\n');
@@ -166,7 +226,7 @@ function getPortFromLocalEnv(localEnv) {
166
226
  }
167
227
 
168
228
  /**
169
- * Gets port from application config file (build.localPort if positive, else port). Uses port-resolver.
229
+ * Gets port from application config file (port only). Uses port-resolver.
170
230
  * @function getPortFromVariablesFile
171
231
  * @param {string} variablesPath - Path to application config
172
232
  * @returns {number|null} Port value or null
@@ -199,7 +259,7 @@ function applyDeveloperIdAdjustment(baseAppPort, devIdNum) {
199
259
 
200
260
  /**
201
261
  * Calculate application port following override chain and developer-id adjustment
202
- * Override chain: env-config.yaml → config.yaml → application.yaml build.localPort → application.yaml port
262
+ * Override chain: env-config.yaml → config.yaml → application.yaml port
203
263
  * @async
204
264
  * @function calculateAppPort
205
265
  * @param {string} [variablesPath] - Path to application config
@@ -212,7 +272,7 @@ async function calculateAppPort(variablesPath, localEnv, envContent, devIdNum) {
212
272
  // Start with env-config value
213
273
  let baseAppPort = getPortFromLocalEnv(localEnv);
214
274
 
215
- // Override with application config build.localPort (strongest)
275
+ // Override with application config port (strongest)
216
276
  const variablesPort = getPortFromVariablesFile(variablesPath);
217
277
  if (variablesPort !== null) {
218
278
  baseAppPort = variablesPort;
@@ -247,13 +307,32 @@ function updateLocalhostUrls(content, baseAppPort, appPort) {
247
307
  }
248
308
 
249
309
  /**
250
- * Adjust infra-related ports in resolved .env content for local environment
251
- * Only handles PORT variable (other ports handled by interpolation)
252
- * Follows flow: getEnvHosts() config.yaml override application config override developer-id adjustment
310
+ * Get the env var name used for PORT in env.template (e.g. PORT=${MISO_PORT} -> MISO_PORT).
311
+ * Used when generating .env for envOutputPath (local, not reload) so we set that var to localPort.
312
+ * @param {string} [variablesPath] - Path to application config (env.template lives in same dir)
313
+ * @returns {string|null} Variable name or null
314
+ */
315
+ function getPortVarFromEnvTemplatePath(variablesPath) {
316
+ if (!variablesPath || !fs.existsSync(variablesPath)) return null;
317
+ const templatePath = path.join(path.dirname(variablesPath), 'env.template');
318
+ if (!fs.existsSync(templatePath)) return null;
319
+ try {
320
+ const content = fs.readFileSync(templatePath, 'utf8');
321
+ const m = content.match(/^PORT\s*=\s*\$\{([A-Za-z0-9_]+)\}/m);
322
+ return m ? m[1] : null;
323
+ } catch {
324
+ return null;
325
+ }
326
+ }
327
+
328
+ /**
329
+ * Adjust infra-related ports in resolved .env content for local environment.
330
+ * Own case: when we generate .env for envOutputPath (not reload), we use localPort (application.yaml build.localPort or port).
331
+ * Sets PORT and the template port var (e.g. MISO_PORT) to localPort so the generated .env is correct for local use.
253
332
  * @async
254
333
  * @function adjustLocalEnvPortsInContent
255
334
  * @param {string} envContent - Resolved .env content
256
- * @param {string} [variablesPath] - Path to application config (to read build.localPort)
335
+ * @param {string} [variablesPath] - Path to application config (to read port and template port var)
257
336
  * @returns {Promise<string>} Updated content with local ports
258
337
  */
259
338
  /**
@@ -348,85 +427,16 @@ async function adjustLocalEnvPortsInContent(envContent, variablesPath) {
348
427
  updated = await rewriteInfraEndpoints(updated, 'local');
349
428
 
350
429
  const envVars = await buildEnvVarsForInterpolation(devIdNum);
430
+ envVars.PORT = String(appPort);
431
+ const portVar = getPortVarFromEnvTemplatePath(variablesPath);
432
+ if (portVar) {
433
+ envVars[portVar] = String(appPort);
434
+ }
351
435
  updated = interpolateEnvVars(updated, envVars);
352
436
 
353
437
  return updated;
354
438
  }
355
439
 
356
- /**
357
- * Read a YAML file and return parsed object
358
- * @function readYamlAtPath
359
- * @param {string} filePath - Absolute file path
360
- * @returns {Object} Parsed YAML object
361
- */
362
- function readYamlAtPath(filePath) {
363
- const content = fs.readFileSync(filePath, 'utf8');
364
- return yaml.load(content);
365
- }
366
-
367
- /**
368
- * Merge a single secret value from canonical into result
369
- * @function mergeSecretValue
370
- * @param {Object} result - Result object to merge into
371
- * @param {string} key - Secret key
372
- * @param {*} canonicalValue - Value from canonical secrets
373
- */
374
- function mergeSecretValue(result, key, canonicalValue) {
375
- const currentValue = result[key];
376
- // Fill missing, empty, or undefined values
377
- if (!(key in result) || currentValue === undefined || currentValue === null || currentValue === '') {
378
- result[key] = canonicalValue;
379
- return;
380
- }
381
- // Only replace values that are encrypted (have secure:// prefix)
382
- // Plaintext values (no secure://) are used as-is
383
- if (typeof currentValue === 'string' && typeof canonicalValue === 'string') {
384
- if (currentValue.startsWith('secure://')) {
385
- result[key] = canonicalValue;
386
- }
387
- }
388
- }
389
-
390
- /**
391
- * Apply canonical secrets path override if configured and file exists
392
- * @async
393
- * @function applyCanonicalSecretsOverride
394
- * @param {Object} currentSecrets - Current secrets map
395
- * @returns {Promise<Object>} Possibly overridden secrets
396
- */
397
- async function applyCanonicalSecretsOverride(currentSecrets) {
398
- let mergedSecrets = currentSecrets || {};
399
- try {
400
- const canonicalPath = await config.getSecretsPath();
401
- if (!canonicalPath) {
402
- return mergedSecrets;
403
- }
404
- const resolvedCanonical = path.isAbsolute(canonicalPath)
405
- ? canonicalPath
406
- : path.resolve(process.cwd(), canonicalPath);
407
- if (!fs.existsSync(resolvedCanonical)) {
408
- return mergedSecrets;
409
- }
410
- const configSecrets = readYamlAtPath(resolvedCanonical);
411
- if (!configSecrets || typeof configSecrets !== 'object') {
412
- return mergedSecrets;
413
- }
414
- // Apply canonical secrets as a fallback source:
415
- // - Do NOT override any existing keys from user/build
416
- // - Add only missing keys from canonical path
417
- // - Also fill in empty/undefined values from canonical path
418
- // - Replace encrypted values (secure://) with canonical plaintext
419
- const result = { ...mergedSecrets };
420
- for (const [key, canonicalValue] of Object.entries(configSecrets)) {
421
- mergeSecretValue(result, key, canonicalValue);
422
- }
423
- mergedSecrets = result;
424
- } catch {
425
- // ignore and fall through
426
- }
427
- return mergedSecrets;
428
- }
429
-
430
440
  /**
431
441
  * Ensure secrets map is non-empty or throw a friendly guidance error
432
442
  * @function ensureNonEmptySecrets
@@ -447,24 +457,8 @@ function ensureNonEmptySecrets(secrets) {
447
457
  * @returns {Object} Validation result
448
458
  */
449
459
  function validateSecrets(envTemplate, secrets) {
450
- const kvPattern = /kv:\/\/([a-zA-Z0-9-_]+)/g;
451
- const missing = [];
452
- const lines = envTemplate.split('\n');
453
- for (const line of lines) {
454
- if (isCommentOrEmptyLine(line)) continue;
455
- let match;
456
- kvPattern.lastIndex = 0;
457
- while ((match = kvPattern.exec(line)) !== null) {
458
- const secretKey = match[1];
459
- if (!(secretKey in secrets)) {
460
- missing.push(`kv://${secretKey}`);
461
- }
462
- }
463
- }
464
- return {
465
- valid: missing.length === 0,
466
- missing
467
- };
460
+ const missing = collectMissingSecrets(envTemplate, secrets);
461
+ return { valid: missing.length === 0, missing };
468
462
  }
469
463
 
470
464
  module.exports = {
@@ -54,8 +54,8 @@ async function getActualSecretsPath(secretsPath, _appName) {
54
54
  };
55
55
  }
56
56
 
57
- // Cascading lookup: user's file first (under configured home)
58
- const userSecretsPath = path.join(paths.getAifabrixHome(), 'secrets.local.yaml');
57
+ // Cascading lookup: user's file first (primary home: AIFABRIX_HOME or ~/.aifabrix)
58
+ const userSecretsPath = path.join(paths.getConfigDirForPaths(), 'secrets.local.yaml');
59
59
 
60
60
  // Check config.yaml for canonical secrets path
61
61
  let buildSecretsPath = null;
@@ -15,6 +15,24 @@ const yaml = require('js-yaml');
15
15
  const logger = require('./logger');
16
16
  const pathsUtil = require('./paths');
17
17
  const { getContainerPort } = require('./port-resolver');
18
+ const { loadYamlTolerantOfDuplicateKeys } = require('./secrets-generator');
19
+
20
+ /**
21
+ * Parses secrets YAML content with fallback for duplicate keys.
22
+ * @param {string} content - Raw file content
23
+ * @returns {Object} Parsed secrets object
24
+ */
25
+ function parseSecretsContent(content) {
26
+ try {
27
+ return yaml.load(content);
28
+ } catch (yamlErr) {
29
+ const msg = yamlErr.message || '';
30
+ if (msg.includes('duplicate') || msg.includes('duplicated mapping')) {
31
+ return loadYamlTolerantOfDuplicateKeys(content);
32
+ }
33
+ throw yamlErr;
34
+ }
35
+ }
18
36
 
19
37
  /**
20
38
  * Loads secrets from file with cascading lookup support
@@ -45,6 +63,38 @@ async function loadSecretsFromFile(filePath) {
45
63
  }
46
64
  }
47
65
 
66
+ /**
67
+ * Loads user secrets from the primary config directory (AIFABRIX_HOME or ~/.aifabrix).
68
+ * Used as the master source when merging with project/public secrets: user values win,
69
+ * missing keys are filled from the public (aifabrix-secrets) file.
70
+ * Does not use config.yaml aifabrix-home so the merge always sees the actual user file.
71
+ *
72
+ * @function loadPrimaryUserSecrets
73
+ * @returns {Object} Loaded secrets object or empty object
74
+ */
75
+ function loadPrimaryUserSecrets() {
76
+ const primaryDir = pathsUtil.getConfigDirForPaths();
77
+ const userSecretsPath = path.join(primaryDir, 'secrets.local.yaml');
78
+ if (!fs.existsSync(userSecretsPath)) {
79
+ return {};
80
+ }
81
+
82
+ try {
83
+ const content = fs.readFileSync(userSecretsPath, 'utf8');
84
+ const secrets = parseSecretsContent(content);
85
+ if (!secrets || typeof secrets !== 'object') {
86
+ throw new Error(`Invalid secrets file format: ${userSecretsPath}`);
87
+ }
88
+ return secrets;
89
+ } catch (error) {
90
+ if (error.message.includes('Invalid secrets file format')) {
91
+ throw error;
92
+ }
93
+ logger.warn(`Warning: Could not read secrets file ${userSecretsPath}: ${error.message}`);
94
+ return {};
95
+ }
96
+ }
97
+
48
98
  /**
49
99
  * Loads user secrets from ~/.aifabrix/secrets.local.yaml
50
100
  * Uses paths.getAifabrixHome() to respect config.yaml aifabrix-home override
@@ -59,7 +109,7 @@ function loadUserSecrets() {
59
109
 
60
110
  try {
61
111
  const content = fs.readFileSync(userSecretsPath, 'utf8');
62
- const secrets = yaml.load(content);
112
+ const secrets = parseSecretsContent(content);
63
113
  if (!secrets || typeof secrets !== 'object') {
64
114
  throw new Error(`Invalid secrets file format: ${userSecretsPath}`);
65
115
  }
@@ -157,6 +207,7 @@ function resolveUrlPort(protocol, hostname, port, urlPath, hostnameToService) {
157
207
 
158
208
  module.exports = {
159
209
  loadSecretsFromFile,
210
+ loadPrimaryUserSecrets,
160
211
  loadUserSecrets,
161
212
  loadDefaultSecrets,
162
213
  buildHostnameToServiceMap,
@@ -0,0 +1,84 @@
1
+ /**
2
+ * AI Fabrix Builder – Secrets file validation
3
+ *
4
+ * Validates secrets.local.yaml (or given path): valid YAML, flat key-value structure,
5
+ * and optional naming convention (*KeyVault suffix per keyvault.md).
6
+ *
7
+ * @fileoverview Secrets file validation for structure and naming
8
+ * @author AI Fabrix Team
9
+ * @version 2.0.0
10
+ */
11
+
12
+ const fs = require('fs');
13
+ const path = require('path');
14
+ const yaml = require('js-yaml');
15
+
16
+ /**
17
+ * Optional naming convention: keys should end with KeyVault or match known patterns.
18
+ * @param {string} key - Secret key
19
+ * @returns {boolean} True if key matches convention
20
+ */
21
+ function keyMatchesNamingConvention(key) {
22
+ if (!key || typeof key !== 'string') return false;
23
+ if (key.endsWith('KeyVault')) return true;
24
+ return /^[a-z0-9-_]+KeyVault$/i.test(key);
25
+ }
26
+
27
+ /**
28
+ * Validate that parsed secrets is a flat object (no nested objects for values).
29
+ * @param {*} parsed - Parsed YAML
30
+ * @param {boolean} checkNaming - Whether to check key naming
31
+ * @returns {string[]} Errors
32
+ */
33
+ function validateParsedSecrets(parsed, checkNaming) {
34
+ const errors = [];
35
+ if (parsed === null || parsed === undefined) return errors;
36
+ if (typeof parsed !== 'object' || Array.isArray(parsed)) {
37
+ errors.push('Secrets file must be a flat key-value object (no nested objects or arrays)');
38
+ return errors;
39
+ }
40
+ for (const [key, value] of Object.entries(parsed)) {
41
+ if (typeof value !== 'string' && typeof value !== 'number' && value !== null && value !== undefined) {
42
+ if (typeof value === 'object') {
43
+ errors.push(`Key "${key}": secret values must be strings or scalars (no nested objects)`);
44
+ }
45
+ }
46
+ if (checkNaming && !keyMatchesNamingConvention(key)) {
47
+ errors.push(`Key "${key}": recommended format is *KeyVault (e.g. postgres-passwordKeyVault)`);
48
+ }
49
+ }
50
+ return errors;
51
+ }
52
+
53
+ /**
54
+ * Validate secrets file at path: YAML syntax, flat object, optional naming check.
55
+ *
56
+ * @param {string} filePath - Path to secrets file
57
+ * @param {Object} [options] - Options
58
+ * @param {boolean} [options.checkNaming=false] - Check key names against *KeyVault convention
59
+ * @returns {{ valid: boolean, errors: string[], path: string }}
60
+ */
61
+ function validateSecretsFile(filePath, options = {}) {
62
+ const checkNaming = Boolean(options.checkNaming);
63
+ if (!filePath || typeof filePath !== 'string') {
64
+ return { valid: false, errors: ['Path is required'], path: '' };
65
+ }
66
+ const resolvedPath = path.isAbsolute(filePath) ? filePath : path.resolve(process.cwd(), filePath);
67
+ if (!fs.existsSync(resolvedPath)) {
68
+ return { valid: false, errors: [`File not found: ${resolvedPath}`], path: resolvedPath };
69
+ }
70
+ let parsed;
71
+ try {
72
+ const content = fs.readFileSync(resolvedPath, 'utf8');
73
+ parsed = yaml.load(content);
74
+ } catch (err) {
75
+ return { valid: false, errors: [`Invalid YAML: ${err.message}`], path: resolvedPath };
76
+ }
77
+ const errors = validateParsedSecrets(parsed, checkNaming);
78
+ return { valid: errors.length === 0, errors, path: resolvedPath };
79
+ }
80
+
81
+ module.exports = {
82
+ validateSecretsFile,
83
+ keyMatchesNamingConvention
84
+ };
@@ -0,0 +1,116 @@
1
+ /**
2
+ * @fileoverview Locate or generate SSH key for Mutagen sync (Windows and Mac). Prefer ed25519.
3
+ * @author AI Fabrix Team
4
+ * @version 2.0.0
5
+ */
6
+
7
+ const fs = require('fs');
8
+ const path = require('path');
9
+ const os = require('os');
10
+ const { execSync } = require('child_process');
11
+
12
+ /**
13
+ * Default SSH directory (user's .ssh)
14
+ * @returns {string} Path to .ssh directory
15
+ */
16
+ function getDefaultSshDir() {
17
+ const home = os.homedir();
18
+ return path.join(home, '.ssh');
19
+ }
20
+
21
+ /**
22
+ * Path to default ed25519 public key
23
+ * @returns {string} Path to id_ed25519.pub
24
+ */
25
+ function getDefaultEd25519PublicKeyPath() {
26
+ return path.join(getDefaultSshDir(), 'id_ed25519.pub');
27
+ }
28
+
29
+ /**
30
+ * Path to default ed25519 private key
31
+ * @returns {string} Path to id_ed25519
32
+ */
33
+ function getDefaultEd25519PrivateKeyPath() {
34
+ return path.join(getDefaultSshDir(), 'id_ed25519');
35
+ }
36
+
37
+ /**
38
+ * Ensure .ssh directory exists
39
+ * @param {string} [sshDir] - SSH directory (default: user .ssh)
40
+ * @returns {string} Resolved SSH dir path
41
+ */
42
+ function ensureSshDir(sshDir) {
43
+ const dir = sshDir || getDefaultSshDir();
44
+ if (!fs.existsSync(dir)) {
45
+ fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
46
+ }
47
+ return dir;
48
+ }
49
+
50
+ /**
51
+ * Generate ed25519 SSH key pair if it does not exist. Idempotent.
52
+ * @param {string} [privateKeyPath] - Path to private key (default: ~/.ssh/id_ed25519)
53
+ * @returns {string} Path to public key file
54
+ * @throws {Error} If ssh-keygen fails
55
+ */
56
+ function ensureEd25519Key(privateKeyPath) {
57
+ const privPath = privateKeyPath || getDefaultEd25519PrivateKeyPath();
58
+ const pubPath = privPath + '.pub';
59
+ if (fs.existsSync(pubPath) && fs.existsSync(privPath)) {
60
+ return pubPath;
61
+ }
62
+ ensureSshDir(path.dirname(privPath));
63
+ execSync(`ssh-keygen -t ed25519 -f "${privPath}" -N "" -C "aifabrix"`, {
64
+ stdio: 'pipe',
65
+ encoding: 'utf8'
66
+ });
67
+ return pubPath;
68
+ }
69
+
70
+ /**
71
+ * Read public key content (single line). Prefer ed25519, fallback to id_rsa.pub.
72
+ * @param {string} [sshDir] - SSH directory
73
+ * @returns {string} Public key line (e.g. "ssh-ed25519 AAAA... aifabrix")
74
+ * @throws {Error} If no key found or read fails
75
+ */
76
+ function readPublicKeyContent(sshDir) {
77
+ const dir = sshDir || getDefaultSshDir();
78
+ const ed25519Pub = path.join(dir, 'id_ed25519.pub');
79
+ const rsaPub = path.join(dir, 'id_rsa.pub');
80
+ let pathToRead = null;
81
+ if (fs.existsSync(ed25519Pub)) {
82
+ pathToRead = ed25519Pub;
83
+ } else if (fs.existsSync(rsaPub)) {
84
+ pathToRead = rsaPub;
85
+ }
86
+ if (!pathToRead) {
87
+ throw new Error(
88
+ 'No SSH public key found. Run: ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -N "" -C "aifabrix"'
89
+ );
90
+ }
91
+ const content = fs.readFileSync(pathToRead, 'utf8').trim();
92
+ const firstLine = content.split('\n')[0];
93
+ if (!firstLine || !firstLine.startsWith('ssh-')) {
94
+ throw new Error(`Invalid SSH public key file: ${pathToRead}`);
95
+ }
96
+ return firstLine;
97
+ }
98
+
99
+ /**
100
+ * Get or create ed25519 key and return its public key content for POST /api/dev/users/:id/ssh-keys.
101
+ * @returns {string} Single-line public key content
102
+ */
103
+ function getOrCreatePublicKeyContent() {
104
+ ensureEd25519Key();
105
+ return readPublicKeyContent();
106
+ }
107
+
108
+ module.exports = {
109
+ getDefaultSshDir,
110
+ getDefaultEd25519PublicKeyPath,
111
+ getDefaultEd25519PrivateKeyPath,
112
+ ensureSshDir,
113
+ ensureEd25519Key,
114
+ readPublicKeyContent,
115
+ getOrCreatePublicKeyContent
116
+ };
@@ -0,0 +1,56 @@
1
+ /**
2
+ * Test log writer - writes debug logs to integration/<appKey>/logs/
3
+ * Sanitization (tokens, secrets) is done by dataplane before responses are returned.
4
+ *
5
+ * @fileoverview Write test request/response logs for debugging
6
+ * @author AI Fabrix Team
7
+ * @version 2.0.0
8
+ */
9
+
10
+ const fs = require('fs').promises;
11
+ const path = require('path');
12
+
13
+ /**
14
+ * Prepare object for JSON serialization (handles circular refs)
15
+ * @param {*} obj - Object to prepare
16
+ * @param {Set} [seen] - Set of seen object references (for circular refs)
17
+ * @returns {*} Copy safe for JSON.stringify
18
+ */
19
+ function sanitizeForLog(obj, seen = new Set()) {
20
+ if (obj === null || obj === undefined || typeof obj !== 'object') return obj;
21
+ if (seen.has(obj)) return '[Circular]';
22
+ seen.add(obj);
23
+ if (Array.isArray(obj)) return obj.map(item => sanitizeForLog(item, seen));
24
+ const out = {};
25
+ for (const [key, value] of Object.entries(obj)) {
26
+ out[key] = sanitizeForLog(value, seen);
27
+ }
28
+ return out;
29
+ }
30
+
31
+ /**
32
+ * Write test log to integration/<appKey>/logs/<logType>-<timestamp>.json
33
+ * @async
34
+ * @param {string} appKey - Application key (used for path)
35
+ * @param {Object} data - Log data (request, response) - will be sanitized
36
+ * @param {string} [logType] - Log type prefix (default: test-integration)
37
+ * @param {string} [integrationBaseDir] - Base dir for integration (default: cwd/integration)
38
+ * @returns {Promise<string>} Path to written file
39
+ * @throws {Error} If write fails
40
+ */
41
+ async function writeTestLog(appKey, data, logType = 'test-integration', integrationBaseDir) {
42
+ const baseDir = integrationBaseDir || path.join(process.cwd(), 'integration');
43
+ const logsDir = path.join(baseDir, appKey, 'logs');
44
+ await fs.mkdir(logsDir, { recursive: true });
45
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
46
+ const filename = `${logType}-${timestamp}.json`;
47
+ const filePath = path.join(logsDir, filename);
48
+ const sanitized = sanitizeForLog(data);
49
+ await fs.writeFile(filePath, JSON.stringify(sanitized, null, 2), 'utf8');
50
+ return filePath;
51
+ }
52
+
53
+ module.exports = {
54
+ sanitizeForLog,
55
+ writeTestLog
56
+ };