@aifabrix/builder 2.40.2 → 2.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/.cursor/rules/docs-rules.mdc +30 -0
  2. package/README.md +7 -5
  3. package/integration/hubspot/README.md +8 -4
  4. package/integration/hubspot/application.json +54 -0
  5. package/integration/hubspot/create-hubspot.js +9 -136
  6. package/integration/hubspot/env.template +3 -4
  7. package/integration/hubspot/hubspot-datasource-company.json +343 -5
  8. package/integration/hubspot/hubspot-datasource-contact.json +413 -5
  9. package/integration/hubspot/hubspot-datasource-deal.json +341 -4
  10. package/integration/hubspot/hubspot-datasource-users.json +116 -0
  11. package/integration/hubspot/hubspot-deploy.json +1250 -108
  12. package/integration/hubspot/hubspot-system.json +15 -32
  13. package/integration/hubspot/test-dataplane-down-tests.js +17 -16
  14. package/integration/hubspot/test-dataplane-down.js +2 -2
  15. package/integration/hubspot/test.js +1 -1
  16. package/jest.config.manual.js +2 -1
  17. package/lib/api/credential.api.js +40 -0
  18. package/lib/api/dev.api.js +423 -0
  19. package/lib/api/external-test.api.js +111 -0
  20. package/lib/api/index.js +42 -19
  21. package/lib/api/pipeline.api.js +66 -120
  22. package/lib/api/types/credential.types.js +23 -0
  23. package/lib/api/types/dev.types.js +140 -0
  24. package/lib/api/types/pipeline.types.js +37 -0
  25. package/lib/api/wizard-platform.api.js +61 -0
  26. package/lib/api/wizard.api.js +34 -1
  27. package/lib/app/config.js +44 -11
  28. package/lib/app/down.js +2 -1
  29. package/lib/app/index.js +12 -1
  30. package/lib/app/prompts.js +44 -29
  31. package/lib/app/push.js +36 -12
  32. package/lib/app/readme.js +9 -6
  33. package/lib/app/run-env-compose.js +264 -0
  34. package/lib/app/run-helpers.js +121 -118
  35. package/lib/app/run.js +148 -28
  36. package/lib/app/show-display.js +1 -1
  37. package/lib/app/show.js +5 -2
  38. package/lib/build/index.js +11 -3
  39. package/lib/cli/setup-app.js +172 -15
  40. package/lib/cli/setup-credential-deployment.js +31 -6
  41. package/lib/cli/setup-dev.js +206 -16
  42. package/lib/cli/setup-environment.js +16 -6
  43. package/lib/cli/setup-external-system.js +89 -24
  44. package/lib/cli/setup-infra.js +82 -15
  45. package/lib/cli/setup-secrets.js +52 -5
  46. package/lib/cli/setup-utility.js +129 -24
  47. package/lib/commands/app-install.js +172 -0
  48. package/lib/commands/app-shell.js +75 -0
  49. package/lib/commands/app-test.js +282 -0
  50. package/lib/commands/app.js +1 -1
  51. package/lib/commands/credential-env.js +162 -0
  52. package/lib/commands/credential-list.js +17 -22
  53. package/lib/commands/credential-push.js +96 -0
  54. package/lib/commands/datasource.js +77 -6
  55. package/lib/commands/dev-cli-handlers.js +141 -0
  56. package/lib/commands/dev-down.js +114 -0
  57. package/lib/commands/dev-init.js +347 -0
  58. package/lib/commands/repair-auth-config.js +99 -0
  59. package/lib/commands/repair-datasource-keys.js +208 -0
  60. package/lib/commands/repair-datasource.js +235 -0
  61. package/lib/commands/repair-env-template.js +348 -0
  62. package/lib/commands/repair-internal.js +85 -0
  63. package/lib/commands/repair-rbac.js +158 -0
  64. package/lib/commands/repair.js +507 -0
  65. package/lib/commands/secrets-list.js +118 -0
  66. package/lib/commands/secrets-remove.js +97 -0
  67. package/lib/commands/secrets-set.js +30 -17
  68. package/lib/commands/secrets-validate.js +50 -0
  69. package/lib/commands/test-e2e-external.js +165 -0
  70. package/lib/commands/up-dataplane.js +2 -2
  71. package/lib/commands/up-miso.js +0 -25
  72. package/lib/commands/upload.js +96 -40
  73. package/lib/commands/wizard-core-helpers.js +226 -4
  74. package/lib/commands/wizard-core.js +67 -29
  75. package/lib/commands/wizard-dataplane.js +1 -1
  76. package/lib/commands/wizard-entity-selection.js +43 -0
  77. package/lib/commands/wizard-headless.js +44 -5
  78. package/lib/commands/wizard-helpers.js +7 -3
  79. package/lib/commands/wizard.js +86 -64
  80. package/lib/core/admin-secrets.js +96 -0
  81. package/lib/core/config.js +7 -1
  82. package/lib/core/secrets-ensure.js +378 -0
  83. package/lib/core/secrets-env-write.js +157 -0
  84. package/lib/core/secrets.js +176 -89
  85. package/lib/datasource/deploy.js +12 -3
  86. package/lib/datasource/field-reference-validator.js +91 -0
  87. package/lib/datasource/test-e2e.js +219 -0
  88. package/lib/datasource/test-integration.js +154 -0
  89. package/lib/datasource/validate.js +21 -3
  90. package/lib/deployment/deployer.js +7 -5
  91. package/lib/deployment/environment-config.js +137 -0
  92. package/lib/deployment/environment.js +21 -98
  93. package/lib/deployment/push.js +32 -2
  94. package/lib/external-system/download.js +188 -203
  95. package/lib/external-system/generator.js +204 -56
  96. package/lib/external-system/test-auth.js +7 -3
  97. package/lib/external-system/test-execution.js +2 -1
  98. package/lib/external-system/test-system-level.js +73 -0
  99. package/lib/external-system/test.js +56 -19
  100. package/lib/generator/external-controller-manifest.js +29 -2
  101. package/lib/generator/external-schema-utils.js +1 -1
  102. package/lib/generator/external.js +10 -3
  103. package/lib/generator/index.js +177 -25
  104. package/lib/generator/split-readme.js +1 -0
  105. package/lib/generator/split-variables.js +7 -1
  106. package/lib/generator/split.js +194 -54
  107. package/lib/generator/wizard-prompts-secondary.js +294 -0
  108. package/lib/generator/wizard-prompts.js +105 -106
  109. package/lib/generator/wizard-readme.js +88 -0
  110. package/lib/generator/wizard.js +155 -158
  111. package/lib/infrastructure/compose.js +11 -1
  112. package/lib/infrastructure/helpers.js +103 -20
  113. package/lib/infrastructure/index.js +98 -12
  114. package/lib/infrastructure/services.js +88 -22
  115. package/lib/schema/application-schema.json +32 -8
  116. package/lib/schema/external-datasource.schema.json +49 -26
  117. package/lib/schema/external-system.schema.json +509 -411
  118. package/lib/schema/wizard-config.schema.json +16 -0
  119. package/lib/utils/api.js +41 -13
  120. package/lib/utils/app-register-auth.js +25 -3
  121. package/lib/utils/auth-headers.js +8 -7
  122. package/lib/utils/cli-utils.js +20 -0
  123. package/lib/utils/compose-generator.js +77 -76
  124. package/lib/utils/compose-handlebars-helpers.js +54 -0
  125. package/lib/utils/compose-vector-helper.js +18 -0
  126. package/lib/utils/config-format-preference.js +51 -0
  127. package/lib/utils/config-format.js +36 -0
  128. package/lib/utils/config-paths.js +127 -2
  129. package/lib/utils/configuration-env-resolver.js +179 -0
  130. package/lib/utils/credential-display.js +83 -0
  131. package/lib/utils/credential-secrets-env.js +357 -0
  132. package/lib/utils/dataplane-pipeline-warning.js +28 -0
  133. package/lib/utils/deployment-validation-helpers.js +4 -4
  134. package/lib/utils/dev-ca-install.js +139 -0
  135. package/lib/utils/dev-cert-helper.js +122 -0
  136. package/lib/utils/device-code-helpers.js +224 -0
  137. package/lib/utils/device-code.js +37 -336
  138. package/lib/utils/docker-build.js +40 -8
  139. package/lib/utils/env-copy.js +103 -13
  140. package/lib/utils/env-map.js +35 -5
  141. package/lib/utils/env-template.js +6 -5
  142. package/lib/utils/error-formatters/http-status-errors.js +20 -2
  143. package/lib/utils/error-formatters/permission-errors.js +0 -1
  144. package/lib/utils/error-formatters/validation-errors.js +0 -1
  145. package/lib/utils/external-readme.js +56 -29
  146. package/lib/utils/external-system-display.js +59 -1
  147. package/lib/utils/external-system-test-helpers.js +21 -8
  148. package/lib/utils/external-system-validators.js +3 -0
  149. package/lib/utils/file-upload.js +20 -50
  150. package/lib/utils/help-builder.js +16 -2
  151. package/lib/utils/infra-status.js +80 -45
  152. package/lib/utils/local-secrets.js +7 -52
  153. package/lib/utils/mutagen-install.js +195 -0
  154. package/lib/utils/mutagen.js +146 -0
  155. package/lib/utils/paths.js +128 -37
  156. package/lib/utils/port-resolver.js +28 -16
  157. package/lib/utils/remote-dev-auth.js +38 -0
  158. package/lib/utils/remote-docker-env.js +43 -0
  159. package/lib/utils/remote-secrets-loader.js +60 -0
  160. package/lib/utils/secrets-canonical.js +93 -0
  161. package/lib/utils/secrets-generator.js +114 -6
  162. package/lib/utils/secrets-helpers.js +108 -114
  163. package/lib/utils/secrets-path.js +2 -2
  164. package/lib/utils/secrets-utils.js +52 -1
  165. package/lib/utils/secrets-validation.js +84 -0
  166. package/lib/utils/ssh-key-helper.js +116 -0
  167. package/lib/utils/test-log-writer.js +56 -0
  168. package/lib/utils/token-manager-messages.js +90 -0
  169. package/lib/utils/token-manager.js +29 -36
  170. package/lib/utils/variable-transformer.js +3 -3
  171. package/lib/validation/env-template-auth.js +157 -0
  172. package/lib/validation/env-template-kv.js +41 -0
  173. package/lib/validation/external-manifest-validator.js +25 -0
  174. package/lib/validation/external-system-auth-rules.js +86 -0
  175. package/lib/validation/validate-batch.js +149 -0
  176. package/lib/validation/validate-datasource-keys-api.js +33 -0
  177. package/lib/validation/validate-display.js +94 -16
  178. package/lib/validation/validate.js +25 -12
  179. package/lib/validation/validator.js +72 -9
  180. package/lib/validation/wizard-datasource-validation.js +50 -0
  181. package/package.json +8 -3
  182. package/scripts/install-local.js +34 -15
  183. package/templates/README.md +0 -1
  184. package/templates/applications/README.md.hbs +4 -4
  185. package/templates/applications/dataplane/application.yaml +6 -5
  186. package/templates/applications/dataplane/env.template +15 -10
  187. package/templates/applications/dataplane/rbac.yaml +2 -2
  188. package/templates/applications/keycloak/env.template +2 -0
  189. package/templates/applications/miso-controller/application.yaml +1 -0
  190. package/templates/applications/miso-controller/env.template +12 -10
  191. package/templates/external-system/README.md.hbs +65 -25
  192. package/templates/external-system/deploy.js.hbs +4 -2
  193. package/templates/external-system/external-datasource.yaml.hbs +217 -0
  194. package/templates/external-system/external-system.json.hbs +1 -18
  195. package/templates/infra/compose.yaml.hbs +6 -0
  196. package/templates/python/docker-compose.hbs +49 -23
  197. package/templates/typescript/docker-compose.hbs +48 -22
  198. package/integration/hubspot/application.yaml +0 -37
@@ -0,0 +1,264 @@
1
+ /**
2
+ * Helpers for application run: clean applications dir, build merged .env, compose safeguard.
3
+ * Keeps run-helpers.js under line limit.
4
+ *
5
+ * @fileoverview Run env and compose helpers
6
+ * @author AI Fabrix Team
7
+ * @version 2.0.0
8
+ */
9
+
10
+ 'use strict';
11
+
12
+ const path = require('path');
13
+ const fs = require('fs').promises;
14
+ const fsSync = require('fs');
15
+ const pathsUtil = require('../utils/paths');
16
+ const adminSecrets = require('../core/admin-secrets');
17
+ const secretsEnvWrite = require('../core/secrets-env-write');
18
+ const { getContainerPort } = require('../utils/port-resolver');
19
+ const { getInfraDirName } = require('../infrastructure/helpers');
20
+
21
+ /**
22
+ * Clean applications directory: remove generated docker-compose.yaml and .env.* files.
23
+ * @param {string|number} developerId - Developer ID
24
+ */
25
+ function cleanApplicationsDir(developerId) {
26
+ const baseDir = pathsUtil.getApplicationsBaseDir(developerId);
27
+ if (!fsSync.existsSync(baseDir)) return;
28
+ const toRemove = [path.join(baseDir, 'docker-compose.yaml')];
29
+ try {
30
+ const entries = fsSync.readdirSync(baseDir);
31
+ for (const name of entries) {
32
+ if (name.startsWith('.env.')) toRemove.push(path.join(baseDir, name));
33
+ }
34
+ } catch {
35
+ // Ignore readdir errors
36
+ }
37
+ for (const filePath of toRemove) {
38
+ try {
39
+ if (fsSync.existsSync(filePath)) fsSync.unlinkSync(filePath);
40
+ } catch {
41
+ // Ignore unlink errors
42
+ }
43
+ }
44
+ }
45
+
46
+ /**
47
+ * Derive PostgreSQL user from database name (same as compose-handlebars-helpers pgUserName).
48
+ * @param {string} dbName - Database name (e.g. keycloak)
49
+ * @returns {string} User name (e.g. keycloak_user)
50
+ */
51
+ function pgUserName(dbName) {
52
+ if (!dbName) return '';
53
+ return `${String(dbName).replace(/-/g, '_')}_user`;
54
+ }
55
+
56
+ /**
57
+ * Inject DB_N_NAME and DB_N_USER from application.yaml databases into env so .env has everything.
58
+ * @param {Object} env - Merged env object (mutated)
59
+ * @param {Object} appConfig - Application config (requires.databases or databases array)
60
+ */
61
+ function injectDatabaseNamesAndUsers(env, appConfig) {
62
+ const databases = appConfig?.requires?.databases || appConfig?.databases;
63
+ if (!Array.isArray(databases) || databases.length === 0) return;
64
+ for (let i = 0; i < databases.length; i++) {
65
+ const db = databases[i];
66
+ const name = db?.name || (appConfig?.app?.key || 'app');
67
+ env[`DB_${i}_NAME`] = name;
68
+ env[`DB_${i}_USER`] = pgUserName(name);
69
+ }
70
+ }
71
+
72
+ /**
73
+ * Get the env var name used for PORT in env.template (e.g. PORT=${MISO_PORT} -> MISO_PORT).
74
+ * @param {string} appName - Application name
75
+ * @returns {string|null} Variable name or null if not found
76
+ */
77
+ function getPortVarFromEnvTemplate(appName) {
78
+ const builderPath = pathsUtil.getBuilderPath(appName);
79
+ const templatePath = path.join(builderPath, 'env.template');
80
+ if (!fsSync.existsSync(templatePath)) return null;
81
+ try {
82
+ const content = fsSync.readFileSync(templatePath, 'utf8');
83
+ const m = content.match(/^PORT\s*=\s*\$\{([A-Za-z0-9_]+)\}/m);
84
+ return m ? m[1] : null;
85
+ } catch {
86
+ return null;
87
+ }
88
+ }
89
+
90
+ /**
91
+ * Override PORT and the template's port variable (e.g. MISO_PORT) with container port from application.yaml.
92
+ * Run .env only: when running in Docker, the app listens on the container port (port or build.containerPort), not localPort.
93
+ * For envOutputPath .env (local, not reload) we use localPort instead - see adjustLocalEnvPortsInContent in secrets-helpers.
94
+ * @param {Object} env - Merged env object (mutated)
95
+ * @param {Object} appConfig - Application configuration (port, build.containerPort)
96
+ * @param {string} appName - Application name (to resolve env.template port var)
97
+ */
98
+ function injectContainerPortForRun(env, appConfig, appName) {
99
+ const containerPort = getContainerPort(appConfig, 3000);
100
+ env.PORT = String(containerPort);
101
+ const portVar = getPortVarFromEnvTemplate(appName);
102
+ if (portVar) {
103
+ env[portVar] = String(containerPort);
104
+ }
105
+ }
106
+
107
+ /** Keys that must never be passed to the app container (admin/start-only). */
108
+ const ADMIN_ONLY_KEYS = [
109
+ 'POSTGRES_PASSWORD',
110
+ 'PGADMIN_DEFAULT_EMAIL',
111
+ 'PGADMIN_DEFAULT_PASSWORD',
112
+ 'REDIS_HOST',
113
+ 'REDIS_COMMANDER_USER',
114
+ 'REDIS_COMMANDER_PASSWORD'
115
+ ];
116
+
117
+ /**
118
+ * Build app-only env (merged minus admin secrets). App container must not receive admin passwords.
119
+ * @param {Object} merged - Full merged env
120
+ * @returns {Object} Env object safe for app container
121
+ */
122
+ function buildAppOnlyEnv(merged) {
123
+ const appOnly = {};
124
+ for (const [k, v] of Object.entries(merged)) {
125
+ if (ADMIN_ONLY_KEYS.includes(k)) continue;
126
+ appOnly[k] = v;
127
+ }
128
+ return appOnly;
129
+ }
130
+
131
+ /**
132
+ * Build env for db-init only: POSTGRES_PASSWORD + DB_N_PASSWORD, DB_N_NAME, DB_N_USER. Used only for start, not in app container.
133
+ * @param {Object} merged - Full merged env
134
+ * @returns {Object} Env object for db-init service only
135
+ */
136
+ function buildDbInitOnlyEnv(merged) {
137
+ const dbInit = {};
138
+ if (merged.POSTGRES_PASSWORD !== undefined) {
139
+ dbInit.POSTGRES_PASSWORD = merged.POSTGRES_PASSWORD;
140
+ }
141
+ for (const [k, v] of Object.entries(merged)) {
142
+ if (k.startsWith('DB_') && (k.endsWith('_PASSWORD') || k.endsWith('_NAME') || k.endsWith('_USER'))) {
143
+ dbInit[k] = v;
144
+ }
145
+ }
146
+ return dbInit;
147
+ }
148
+
149
+ /**
150
+ * Return pgpass paths under infra-dev* directories in aifabrix home (for fallback lookup).
151
+ * @param {string} aifabrixDir - Aifabrix home directory
152
+ * @returns {string[]} Paths to pgpass files
153
+ */
154
+ function getInfraDevPgpassPaths(aifabrixDir) {
155
+ if (!fsSync.existsSync(aifabrixDir)) return [];
156
+ let entries;
157
+ try {
158
+ entries = fsSync.readdirSync(aifabrixDir).sort();
159
+ } catch {
160
+ return [];
161
+ }
162
+ return entries
163
+ .filter((name) => name.startsWith('infra-dev'))
164
+ .map((name) => path.join(aifabrixDir, name, 'pgpass'));
165
+ }
166
+
167
+ /**
168
+ * Read first password from a pgpass file (format host:port:db:user:password).
169
+ * @param {string} pgpassPath - Path to pgpass file
170
+ * @returns {Promise<string|undefined>} Password or undefined
171
+ */
172
+ async function readPasswordFromPgpassFile(pgpassPath) {
173
+ const content = await fs.readFile(pgpassPath, 'utf8');
174
+ const line = content.split('\n')[0];
175
+ if (!line) return undefined;
176
+ const parts = line.split(':');
177
+ return parts.length >= 5 ? parts[4].trim() : undefined;
178
+ }
179
+
180
+ /**
181
+ * Read POSTGRES_PASSWORD from an existing infra pgpass so db-init uses the same password as running Postgres.
182
+ * Tries dev-specific, then default infra, then any infra-dev* dir (e.g. dev 1 run when only infra-dev06 has pgpass).
183
+ * @param {number|string} developerId - Developer ID
184
+ * @returns {Promise<string|undefined>} Password or undefined
185
+ */
186
+ async function readPostgresPasswordFromPgpass(developerId) {
187
+ const home = pathsUtil.getAifabrixHome();
188
+ const idNum = typeof developerId === 'string' ? parseInt(developerId, 10) : developerId;
189
+ const candidates = [path.join(home, getInfraDirName(developerId), 'pgpass')];
190
+ if (idNum !== 0) candidates.push(path.join(home, getInfraDirName(0), 'pgpass'));
191
+ const extra = getInfraDevPgpassPaths(home).filter((p) => !candidates.includes(p));
192
+ candidates.push(...extra);
193
+ for (const pgpassPath of candidates) {
194
+ if (!fsSync.existsSync(pgpassPath)) continue;
195
+ try {
196
+ const pwd = await readPasswordFromPgpassFile(pgpassPath);
197
+ if (pwd !== undefined) return pwd;
198
+ } catch {
199
+ // ignore
200
+ }
201
+ }
202
+ return undefined;
203
+ }
204
+
205
+ /**
206
+ * Build two run env files: .env.run (app-only, no admin secrets) and .env.run.admin (start-only, for db-init).
207
+ * Admin password is never set in the app container; .env.run.admin is used only for start and then deleted.
208
+ * When an infra pgpass exists, POSTGRES_PASSWORD is taken from it so db-init matches the running Postgres.
209
+ * @async
210
+ * @param {string} appName - Application name
211
+ * @param {Object} appConfig - Application configuration
212
+ * @param {string} devDir - Applications directory path
213
+ * @param {number|string} [developerId] - Developer ID (for pgpass lookup)
214
+ * @returns {Promise<{ runEnvPath: string, runEnvAdminPath: string }>} Paths to .env.run and .env.run.admin
215
+ */
216
+ async function buildMergedRunEnvAndWrite(appName, appConfig, devDir, developerId) {
217
+ const infra = require('../infrastructure');
218
+ const ensureAdminSecretsFn = typeof infra.ensureAdminSecrets === 'function'
219
+ ? infra.ensureAdminSecrets
220
+ : require('../infrastructure/helpers').ensureAdminSecrets;
221
+ await ensureAdminSecretsFn();
222
+ const adminObj = await adminSecrets.readAndDecryptAdminSecrets();
223
+ const appObj = await secretsEnvWrite.resolveAndGetEnvMap(appName, {
224
+ environment: 'docker',
225
+ secretsPath: null,
226
+ force: false
227
+ });
228
+ const merged = { ...adminObj, ...appObj };
229
+ if (developerId !== undefined) {
230
+ const pgpassPwd = await readPostgresPasswordFromPgpass(developerId);
231
+ if (pgpassPwd !== undefined) merged.POSTGRES_PASSWORD = pgpassPwd;
232
+ }
233
+ injectDatabaseNamesAndUsers(merged, appConfig);
234
+ injectContainerPortForRun(merged, appConfig, appName);
235
+
236
+ const runEnvPath = path.join(devDir, '.env.run');
237
+ const runEnvAdminPath = path.join(devDir, '.env.run.admin');
238
+
239
+ const appOnly = buildAppOnlyEnv(merged);
240
+ const dbInitOnly = buildDbInitOnlyEnv(merged);
241
+
242
+ await fs.writeFile(runEnvPath, adminSecrets.envObjectToContent(appOnly), { mode: 0o600 });
243
+ await fs.writeFile(runEnvAdminPath, adminSecrets.envObjectToContent(dbInitOnly), { mode: 0o600 });
244
+
245
+ return { runEnvPath, runEnvAdminPath };
246
+ }
247
+
248
+ /**
249
+ * Assert generated compose does not contain password literals in environment (ISO 27K).
250
+ * @param {string} composeContent - Generated docker-compose content
251
+ * @throws {Error} If password keys appear in environment-like assignment
252
+ */
253
+ function assertNoPasswordLiteralsInCompose(composeContent) {
254
+ const badPattern = /\n\s+(-?\s*)(POSTGRES_PASSWORD|DB_\d+_PASSWORD)\s*[:=]/;
255
+ if (badPattern.test(composeContent)) {
256
+ throw new Error('Generated compose must not contain password literals (POSTGRES_PASSWORD, DB_*_PASSWORD). Use env_file only.');
257
+ }
258
+ }
259
+
260
+ module.exports = {
261
+ cleanApplicationsDir,
262
+ buildMergedRunEnvAndWrite,
263
+ assertNoPasswordLiteralsInCompose
264
+ };
@@ -17,8 +17,6 @@ const { exec } = require('child_process');
17
17
  const { loadConfigFile } = require('../utils/config-format');
18
18
  const { promisify } = require('util');
19
19
  const validator = require('../validation/validator');
20
- const infra = require('../infrastructure');
21
- const secrets = require('../core/secrets');
22
20
  const config = require('../core/config');
23
21
  const buildCopy = require('../utils/build-copy');
24
22
  const logger = require('../utils/logger');
@@ -27,7 +25,10 @@ const composeGenerator = require('../utils/compose-generator');
27
25
  const dockerUtils = require('../utils/docker');
28
26
  const containerHelpers = require('../utils/app-run-containers');
29
27
  const pathsUtil = require('../utils/paths');
28
+ const runEnvCompose = require('./run-env-compose');
29
+ const { resolveEnvOutputPath, writeEnvOutputForReload, writeEnvOutputForLocal } = require('../utils/env-copy');
30
30
  const { resolveVersionForApp } = require('../utils/image-version');
31
+ const { parseImageOverride } = require('../utils/parse-image-ref');
31
32
 
32
33
  const execAsync = promisify(exec);
33
34
 
@@ -160,6 +161,28 @@ async function resolveAndUpdateVersion(appName, appConfig, debug) {
160
161
  }
161
162
  }
162
163
 
164
+ /**
165
+ * Resolve image name and tag from app config and optional run override.
166
+ * @param {string} appName - Application name
167
+ * @param {Object} appConfig - Application configuration
168
+ * @param {Object} runOptions - Run options; runOptions.image overrides config
169
+ * @returns {{ imageName: string, imageTag: string }} imageName and imageTag
170
+ */
171
+ function resolveRunImage(appName, appConfig, runOptions) {
172
+ const imageOverride = runOptions && runOptions.image;
173
+ if (imageOverride) {
174
+ const parsed = parseImageOverride(imageOverride);
175
+ return {
176
+ imageName: parsed ? parsed.name : composeGenerator.getImageName(appConfig, appName),
177
+ imageTag: parsed ? parsed.tag : (appConfig.image && appConfig.image.tag) || 'latest'
178
+ };
179
+ }
180
+ return {
181
+ imageName: composeGenerator.getImageName(appConfig, appName),
182
+ imageTag: (appConfig.image && appConfig.image.tag) || 'latest'
183
+ };
184
+ }
185
+
163
186
  /**
164
187
  * Checks prerequisites: Docker image and (optionally) infrastructure
165
188
  * @async
@@ -167,11 +190,11 @@ async function resolveAndUpdateVersion(appName, appConfig, debug) {
167
190
  * @param {Object} appConfig - Application configuration
168
191
  * @param {boolean} [debug=false] - Enable debug logging
169
192
  * @param {boolean} [skipInfraCheck=false] - When true, skip infra health check (e.g. when caller already verified, e.g. up-miso)
193
+ * @param {Object} [runOptions] - Run options; when runOptions.image is set, that image is checked instead of config-derived
170
194
  * @throws {Error} If prerequisites are not met
171
195
  */
172
- async function checkPrerequisites(appName, appConfig, debug = false, skipInfraCheck = false) {
173
- const imageName = composeGenerator.getImageName(appConfig, appName);
174
- const imageTag = appConfig.image?.tag || 'latest';
196
+ async function checkPrerequisites(appName, appConfig, debug = false, skipInfraCheck = false, runOptions = {}) {
197
+ const { imageName, imageTag } = resolveRunImage(appName, appConfig, runOptions);
175
198
  const fullImageName = `${imageName}:${imageTag}`;
176
199
 
177
200
  if (debug) {
@@ -181,7 +204,11 @@ async function checkPrerequisites(appName, appConfig, debug = false, skipInfraCh
181
204
  logger.log(chalk.blue(`Checking if image ${fullImageName} exists...`));
182
205
  const imageExists = await checkImageExists(imageName, imageTag, debug);
183
206
  if (!imageExists) {
184
- throw new Error(`Docker image ${fullImageName} not found\nRun 'aifabrix build ${appName}' first`);
207
+ const isTemplateApp = TEMPLATE_APP_KEYS.includes(appName);
208
+ const hint = isTemplateApp
209
+ ? `Pull the image (e.g. docker pull ${fullImageName}) or use --image ${appName}=<image> for up-miso/up-dataplane.`
210
+ : `Run 'aifabrix build ${appName}' first`;
211
+ throw new Error(`Docker image ${fullImageName} not found\n${hint}`);
185
212
  }
186
213
  logger.log(chalk.green(`✓ Image ${fullImageName} found`));
187
214
 
@@ -200,6 +227,7 @@ async function checkPrerequisites(appName, appConfig, debug = false, skipInfraCh
200
227
  */
201
228
  async function checkInfraHealthOrThrow(debug) {
202
229
  logger.log(chalk.blue('Checking infrastructure health...'));
230
+ const infra = require('../infrastructure');
203
231
  const infraHealth = await infra.checkInfraHealth();
204
232
  if (debug) {
205
233
  logger.log(chalk.gray(`[DEBUG] Infrastructure health: ${JSON.stringify(infraHealth, null, 2)}`));
@@ -230,73 +258,20 @@ async function ensureDevDirectory(appName, developerId) {
230
258
  }
231
259
 
232
260
  /**
233
- * Generate or update .env file for Docker
234
- * @async
235
- * @param {string} appName - Application name
236
- * @param {string} builderEnvPath - Path to builder .env file
237
- * @param {boolean} [skipOutputPath=false] - When true, skip copying to envOutputPath (e.g. up-miso/up-dataplane, no local code)
238
- */
239
- async function ensureEnvFile(appName, builderEnvPath, skipOutputPath = false) {
240
- if (!fsSync.existsSync(builderEnvPath)) {
241
- logger.log(chalk.yellow('Generating .env file from template...'));
242
- await secrets.generateEnvFile(appName, null, 'docker', false, skipOutputPath);
243
- } else {
244
- logger.log(chalk.blue('Updating .env file for Docker environment...'));
245
- await secrets.generateEnvFile(appName, null, 'docker', false, skipOutputPath);
246
- }
247
- }
248
-
249
- /**
250
- * Copy .env file to dev directory
251
- * @async
252
- * @param {string} builderEnvPath - Path to builder .env file
253
- * @param {string} devEnvPath - Path to dev .env file
254
- */
255
- async function copyEnvToDev(builderEnvPath, devEnvPath) {
256
- if (fsSync.existsSync(builderEnvPath)) {
257
- await fs.copyFile(builderEnvPath, devEnvPath);
258
- }
259
- }
260
-
261
- /**
262
- * Handle envOutputPath configuration
263
- * @async
264
- * @param {string} appName - Application name
265
- * @param {string} configPath - Path to application config file
266
- * @param {string} builderEnvPath - Path to builder .env file
267
- * @param {string} devEnvPath - Path to dev .env file
268
- * @param {boolean} [skipOutputPath=false] - When true, skip (e.g. up-miso/up-dataplane, no local code)
269
- */
270
- async function handleEnvOutputPath(appName, configPath, builderEnvPath, devEnvPath, skipOutputPath = false) {
271
- if (skipOutputPath) {
272
- return;
273
- }
274
- let variables;
275
- try {
276
- variables = loadConfigFile(configPath);
277
- } catch {
278
- return;
279
- }
280
-
281
- if (variables?.build?.envOutputPath && variables.build.envOutputPath !== null) {
282
- logger.log(chalk.blue('Ensuring .env file in apps/ directory is updated for Docker...'));
283
- await secrets.generateEnvFile(appName, null, 'docker');
284
- await copyEnvToDev(builderEnvPath, devEnvPath);
285
- }
286
- }
287
-
288
- /**
289
- * Calculate compose port from options or app config
290
- * @param {Object} options - Run options
261
+ * Calculate host port for docker-compose mapping (first port in "host:container").
262
+ * Uses application.yaml top-level port (not localPort). Second port is always containerPort from config.
263
+ * Example: keycloak port 8082, containerPort 8080 → "8082:8080"; miso-controller port 3000 → "3000:3000".
264
+ *
265
+ * @param {Object} options - Run options (may include port override)
291
266
  * @param {Object} appConfig - Application configuration
292
267
  * @param {string} developerId - Developer ID
293
- * @returns {number} Port number
268
+ * @returns {number} Host port number
294
269
  */
295
270
  function calculateComposePort(options, appConfig, developerId) {
296
271
  if (options.port) {
297
272
  return options.port;
298
273
  }
299
- const basePort = appConfig.port || 3000;
274
+ const basePort = appConfig.port ?? 3000;
300
275
  const idNum = typeof developerId === 'string' ? parseInt(developerId, 10) : developerId;
301
276
  return idNum === 0 ? basePort : basePort + (idNum * 100);
302
277
  }
@@ -313,84 +288,104 @@ function calculateComposePort(options, appConfig, developerId) {
313
288
  async function generateComposeFile(appName, appConfig, composeOptions, devDir) {
314
289
  logger.log(chalk.blue('Generating Docker Compose configuration...'));
315
290
  const composeContent = await composeGenerator.generateDockerCompose(appName, appConfig, composeOptions);
291
+ runEnvCompose.assertNoPasswordLiteralsInCompose(composeContent);
316
292
  const tempComposePath = path.join(devDir, 'docker-compose.yaml');
317
293
  await fs.writeFile(tempComposePath, composeContent);
318
294
  return tempComposePath;
319
295
  }
320
296
 
321
297
  /**
322
- * Prepares environment: ensures .env file and generates Docker Compose
298
+ * Writes .env to envOutputPath when application.yaml build.envOutputPath is set.
323
299
  * @async
324
300
  * @param {string} appName - Application name
325
301
  * @param {Object} appConfig - Application configuration
326
- * @param {Object} options - Run options
327
- * @returns {Promise<string>} Path to generated compose file
302
+ * @param {string} runEnvPath - Path to .env.run
303
+ * @param {Object} options - Run options (reload flag)
304
+ */
305
+ async function writeEnvOutputIfConfigured(appName, appConfig, runEnvPath, options) {
306
+ if (options && options.skipEnvOutputPath === true) return;
307
+ const envOutputPathRaw = appConfig.build?.envOutputPath;
308
+ if (!envOutputPathRaw || typeof envOutputPathRaw !== 'string' || envOutputPathRaw.trim() === '') {
309
+ return;
310
+ }
311
+ const configPath = path.join(pathsUtil.getBuilderPath(appName), 'application.yaml');
312
+ const outputPath = resolveEnvOutputPath(envOutputPathRaw.trim(), configPath);
313
+ const outputDir = path.dirname(outputPath);
314
+ if (!fsSync.existsSync(outputDir)) {
315
+ await fs.mkdir(outputDir, { recursive: true });
316
+ }
317
+ if (options.reload) {
318
+ await writeEnvOutputForReload(outputPath, runEnvPath);
319
+ } else {
320
+ await writeEnvOutputForLocal(appName, outputPath);
321
+ }
322
+ }
323
+
324
+ /**
325
+ * Prepares environment: clean applications dir, build two .env files (app-only + start-only), generate Docker Compose.
326
+ * .env.run = app container only (no admin secrets). .env.run.admin = db-init/start only (POSTGRES_PASSWORD etc.), deleted after run.
327
+ *
328
+ * @async
329
+ * @param {string} appName - Application name
330
+ * @param {Object} appConfig - Application configuration
331
+ * @param {Object} options - Run options (may include envFilePath, devMountPath from caller)
332
+ * @returns {Promise<{ composePath: string, runEnvPath: string, runEnvAdminPath: string }>} Paths to compose and both run .env files (delete after success)
328
333
  */
329
334
  async function prepareEnvironment(appName, appConfig, options) {
330
335
  const developerId = await config.getDeveloperId();
331
336
  const devDir = await ensureDevDirectory(appName, developerId);
332
- const skipEnvOutputPath = options.skipEnvOutputPath === true;
333
337
 
334
- // Generate/update .env file (respect AIFABRIX_BUILDER_DIR when set by up-miso/up-dataplane)
335
- const builderEnvPath = path.join(pathsUtil.getBuilderPath(appName), '.env');
336
- await ensureEnvFile(appName, builderEnvPath, skipEnvOutputPath);
338
+ runEnvCompose.cleanApplicationsDir(developerId);
339
+ logger.log(chalk.blue('Building merged .env (admin + app secrets)...'));
340
+ const { runEnvPath, runEnvAdminPath } = await runEnvCompose.buildMergedRunEnvAndWrite(appName, appConfig, devDir, developerId);
337
341
 
338
- // Copy .env to dev directory
339
- const devEnvPath = path.join(devDir, '.env');
340
- await copyEnvToDev(builderEnvPath, devEnvPath);
342
+ const composeOptions = {
343
+ ...options,
344
+ envFilePath: runEnvPath,
345
+ dbInitEnvFilePath: runEnvAdminPath
346
+ };
347
+ composeOptions.port = calculateComposePort(composeOptions, appConfig, developerId);
348
+ const composePath = await generateComposeFile(appName, appConfig, composeOptions, devDir);
341
349
 
342
- // Handle envOutputPath if configured (skipped when skipEnvOutputPath e.g. up-miso/up-dataplane)
343
- let configPath;
344
- try {
345
- configPath = pathsUtil.resolveApplicationConfigPath(devDir);
346
- } catch {
347
- configPath = null;
348
- }
349
- if (configPath) {
350
- await handleEnvOutputPath(appName, configPath, builderEnvPath, devEnvPath, skipEnvOutputPath);
351
- }
350
+ await writeEnvOutputIfConfigured(appName, appConfig, runEnvPath, options);
352
351
 
353
- // Generate Docker Compose
354
- const composeOptions = { ...options };
355
- composeOptions.port = calculateComposePort(composeOptions, appConfig, developerId);
356
- return await generateComposeFile(appName, appConfig, composeOptions, devDir);
352
+ return { composePath, runEnvPath, runEnvAdminPath };
357
353
  }
358
354
 
359
355
  /**
360
- * Prepare environment variables from admin secrets
356
+ * Prepare environment variables for docker compose (no secrets in host env; compose uses env_file).
361
357
  * @async
362
358
  * @param {boolean} debug - Enable debug logging
363
- * @returns {Promise<Object>} Environment variables object
359
+ * @returns {Promise<Object>} Environment variables object for child process
364
360
  */
365
361
  async function prepareContainerEnv(debug) {
366
- const adminSecretsPath = await infra.ensureAdminSecrets();
367
- if (debug) {
368
- logger.log(chalk.gray(`[DEBUG] Admin secrets path: ${adminSecretsPath}`));
369
- }
362
+ const env = { ...process.env };
370
363
 
371
- const adminSecretsContent = fsSync.readFileSync(adminSecretsPath, 'utf8');
372
- const postgresPasswordMatch = adminSecretsContent.match(/^POSTGRES_PASSWORD=(.+)$/m);
373
- const postgresPassword = postgresPasswordMatch ? postgresPasswordMatch[1] : '';
364
+ if (typeof process.getuid === 'function' && typeof process.getgid === 'function') {
365
+ env.AIFABRIX_UID = String(process.getuid());
366
+ env.AIFABRIX_GID = String(process.getgid());
367
+ } else {
368
+ env.AIFABRIX_UID = '1000';
369
+ env.AIFABRIX_GID = '1000';
370
+ }
374
371
 
375
- const env = {
376
- ...process.env,
377
- ADMIN_SECRETS_PATH: adminSecretsPath,
378
- POSTGRES_PASSWORD: postgresPassword
379
- };
372
+ const { getRemoteDockerEnv } = require('../utils/remote-docker-env');
373
+ const remoteDocker = await getRemoteDockerEnv();
374
+ Object.assign(env, remoteDocker);
380
375
 
381
376
  if (debug) {
382
- logger.log(chalk.gray(`[DEBUG] Environment variables: ADMIN_SECRETS_PATH=${adminSecretsPath}, POSTGRES_PASSWORD=${postgresPassword ? '***' : '(not set)'}`));
377
+ logger.log(chalk.gray('[DEBUG] Container env prepared (secrets via env_file)'));
383
378
  }
384
379
 
385
380
  return env;
386
381
  }
387
382
 
388
383
  /**
389
- * Execute docker-compose up command
384
+ * Execute docker-compose up command. Services get env from env_file in the compose (e.g. .env.run).
390
385
  * @async
391
386
  * @param {string} composeCmdBase - Base compose command
392
387
  * @param {string} composePath - Path to compose file
393
- * @param {Object} env - Environment variables
388
+ * @param {Object} env - Environment variables for the child process
394
389
  * @param {boolean} debug - Enable debug logging
395
390
  */
396
391
  async function executeComposeUp(composeCmdBase, composePath, env, debug) {
@@ -403,37 +398,44 @@ async function executeComposeUp(composeCmdBase, composePath, env, debug) {
403
398
  }
404
399
 
405
400
  /**
406
- * Starts the container and waits for health check
401
+ * Starts the container and waits for health check. Deletes run .env files after success (ISO 27K).
407
402
  * @async
408
403
  * @param {string} appName - Application name
409
404
  * @param {string} composePath - Path to Docker Compose file
410
405
  * @param {number} port - Application port
411
406
  * @param {Object} appConfig - Application configuration
412
- * @param {boolean} [debug=false] - Enable debug logging
407
+ * @param {Object} [opts] - Options
408
+ * @param {boolean} [opts.debug=false] - Enable debug logging
409
+ * @param {string|null} [opts.runEnvPath=null] - Path to .env.run (app-only) to delete after successful start
410
+ * @param {string|null} [opts.runEnvAdminPath=null] - Path to .env.run.admin (start-only) to delete after successful start
413
411
  * @throws {Error} If container fails to start or become healthy
414
412
  */
415
- async function startContainer(appName, composePath, port, appConfig = null, debug = false) {
413
+ async function startContainer(appName, composePath, port, appConfig = null, opts = {}) {
414
+ const { debug = false, runEnvPath = null, runEnvAdminPath = null } = opts;
416
415
  logger.log(chalk.blue(`Starting ${appName}...`));
417
416
 
418
- // Ensure Docker + Compose available and determine correct compose command
419
417
  const composeCmdBase = await dockerUtils.ensureDockerAndCompose().then(() => dockerUtils.getComposeCommand());
420
-
421
- // Prepare environment variables
422
418
  const env = await prepareContainerEnv(debug);
423
-
424
- // Execute compose up
425
419
  await executeComposeUp(composeCmdBase, composePath, env, debug);
426
420
 
427
- // Get container name and log status
428
421
  const idNum = typeof appConfig.developerId === 'string' ? parseInt(appConfig.developerId, 10) : appConfig.developerId;
429
422
  const containerName = idNum === 0 ? `aifabrix-${appName}` : `aifabrix-dev${appConfig.developerId}-${appName}`;
430
423
  logger.log(chalk.green(`✓ Container ${containerName} started`));
431
424
  await containerHelpers.logContainerStatus(containerName, debug);
432
425
 
433
- // Wait for health check
434
426
  const healthCheckPath = appConfig?.healthCheck?.path || '/health';
435
427
  logger.log(chalk.blue(`Waiting for application to be healthy at http://localhost:${port}${healthCheckPath}...`));
436
428
  await waitForHealthCheck(appName, 90, port, appConfig, debug);
429
+
430
+ for (const p of [runEnvPath, runEnvAdminPath]) {
431
+ if (p && typeof p === 'string') {
432
+ try {
433
+ await fs.unlink(p);
434
+ } catch (err) {
435
+ if (err.code !== 'ENOENT') logger.log(chalk.yellow(`Warning: could not remove run .env: ${err.message}`));
436
+ }
437
+ }
438
+ }
437
439
  }
438
440
 
439
441
  /**
@@ -461,6 +463,7 @@ module.exports = {
461
463
  checkPrerequisites,
462
464
  prepareEnvironment,
463
465
  startContainer,
464
- displayRunStatus
466
+ displayRunStatus,
467
+ cleanApplicationsDir: runEnvCompose.cleanApplicationsDir
465
468
  };
466
469