@aifabrix/builder 2.40.2 ā 2.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursor/rules/docs-rules.mdc +30 -0
- package/README.md +7 -5
- package/integration/hubspot/README.md +8 -4
- package/integration/hubspot/application.json +54 -0
- package/integration/hubspot/create-hubspot.js +9 -136
- package/integration/hubspot/env.template +3 -4
- package/integration/hubspot/hubspot-datasource-company.json +343 -5
- package/integration/hubspot/hubspot-datasource-contact.json +413 -5
- package/integration/hubspot/hubspot-datasource-deal.json +341 -4
- package/integration/hubspot/hubspot-datasource-users.json +116 -0
- package/integration/hubspot/hubspot-deploy.json +1250 -108
- package/integration/hubspot/hubspot-system.json +15 -32
- package/integration/hubspot/test-dataplane-down-tests.js +17 -16
- package/integration/hubspot/test-dataplane-down.js +2 -2
- package/integration/hubspot/test.js +1 -1
- package/jest.config.manual.js +2 -1
- package/lib/api/credential.api.js +40 -0
- package/lib/api/dev.api.js +423 -0
- package/lib/api/external-test.api.js +111 -0
- package/lib/api/index.js +42 -19
- package/lib/api/pipeline.api.js +66 -120
- package/lib/api/types/credential.types.js +23 -0
- package/lib/api/types/dev.types.js +140 -0
- package/lib/api/types/pipeline.types.js +37 -0
- package/lib/api/wizard-platform.api.js +61 -0
- package/lib/api/wizard.api.js +34 -1
- package/lib/app/config.js +44 -11
- package/lib/app/down.js +2 -1
- package/lib/app/index.js +12 -1
- package/lib/app/prompts.js +44 -29
- package/lib/app/push.js +36 -12
- package/lib/app/readme.js +9 -6
- package/lib/app/run-env-compose.js +264 -0
- package/lib/app/run-helpers.js +121 -118
- package/lib/app/run.js +148 -28
- package/lib/app/show-display.js +1 -1
- package/lib/app/show.js +5 -2
- package/lib/build/index.js +11 -3
- package/lib/cli/setup-app.js +172 -15
- package/lib/cli/setup-credential-deployment.js +31 -6
- package/lib/cli/setup-dev.js +206 -16
- package/lib/cli/setup-environment.js +16 -6
- package/lib/cli/setup-external-system.js +89 -24
- package/lib/cli/setup-infra.js +82 -15
- package/lib/cli/setup-secrets.js +52 -5
- package/lib/cli/setup-utility.js +129 -24
- package/lib/commands/app-install.js +172 -0
- package/lib/commands/app-shell.js +75 -0
- package/lib/commands/app-test.js +282 -0
- package/lib/commands/app.js +1 -1
- package/lib/commands/credential-env.js +162 -0
- package/lib/commands/credential-list.js +17 -22
- package/lib/commands/credential-push.js +96 -0
- package/lib/commands/datasource.js +77 -6
- package/lib/commands/dev-cli-handlers.js +141 -0
- package/lib/commands/dev-down.js +114 -0
- package/lib/commands/dev-init.js +347 -0
- package/lib/commands/repair-auth-config.js +99 -0
- package/lib/commands/repair-datasource-keys.js +208 -0
- package/lib/commands/repair-datasource.js +235 -0
- package/lib/commands/repair-env-template.js +348 -0
- package/lib/commands/repair-internal.js +85 -0
- package/lib/commands/repair-rbac.js +158 -0
- package/lib/commands/repair.js +507 -0
- package/lib/commands/secrets-list.js +118 -0
- package/lib/commands/secrets-remove.js +97 -0
- package/lib/commands/secrets-set.js +30 -17
- package/lib/commands/secrets-validate.js +50 -0
- package/lib/commands/test-e2e-external.js +165 -0
- package/lib/commands/up-dataplane.js +2 -2
- package/lib/commands/up-miso.js +0 -25
- package/lib/commands/upload.js +96 -40
- package/lib/commands/wizard-core-helpers.js +226 -4
- package/lib/commands/wizard-core.js +67 -29
- package/lib/commands/wizard-dataplane.js +1 -1
- package/lib/commands/wizard-entity-selection.js +43 -0
- package/lib/commands/wizard-headless.js +44 -5
- package/lib/commands/wizard-helpers.js +7 -3
- package/lib/commands/wizard.js +86 -64
- package/lib/core/admin-secrets.js +96 -0
- package/lib/core/config.js +7 -1
- package/lib/core/secrets-ensure.js +378 -0
- package/lib/core/secrets-env-write.js +157 -0
- package/lib/core/secrets.js +176 -89
- package/lib/datasource/deploy.js +12 -3
- package/lib/datasource/field-reference-validator.js +91 -0
- package/lib/datasource/test-e2e.js +219 -0
- package/lib/datasource/test-integration.js +154 -0
- package/lib/datasource/validate.js +21 -3
- package/lib/deployment/deployer.js +7 -5
- package/lib/deployment/environment-config.js +137 -0
- package/lib/deployment/environment.js +21 -98
- package/lib/deployment/push.js +32 -2
- package/lib/external-system/download.js +188 -203
- package/lib/external-system/generator.js +204 -56
- package/lib/external-system/test-auth.js +7 -3
- package/lib/external-system/test-execution.js +2 -1
- package/lib/external-system/test-system-level.js +73 -0
- package/lib/external-system/test.js +56 -19
- package/lib/generator/external-controller-manifest.js +29 -2
- package/lib/generator/external-schema-utils.js +1 -1
- package/lib/generator/external.js +10 -3
- package/lib/generator/index.js +177 -25
- package/lib/generator/split-readme.js +1 -0
- package/lib/generator/split-variables.js +7 -1
- package/lib/generator/split.js +194 -54
- package/lib/generator/wizard-prompts-secondary.js +294 -0
- package/lib/generator/wizard-prompts.js +105 -106
- package/lib/generator/wizard-readme.js +88 -0
- package/lib/generator/wizard.js +155 -158
- package/lib/infrastructure/compose.js +11 -1
- package/lib/infrastructure/helpers.js +103 -20
- package/lib/infrastructure/index.js +98 -12
- package/lib/infrastructure/services.js +88 -22
- package/lib/schema/application-schema.json +32 -8
- package/lib/schema/external-datasource.schema.json +49 -26
- package/lib/schema/external-system.schema.json +509 -411
- package/lib/schema/wizard-config.schema.json +16 -0
- package/lib/utils/api.js +41 -13
- package/lib/utils/app-register-auth.js +25 -3
- package/lib/utils/auth-headers.js +8 -7
- package/lib/utils/cli-utils.js +20 -0
- package/lib/utils/compose-generator.js +77 -76
- package/lib/utils/compose-handlebars-helpers.js +54 -0
- package/lib/utils/compose-vector-helper.js +18 -0
- package/lib/utils/config-format-preference.js +51 -0
- package/lib/utils/config-format.js +36 -0
- package/lib/utils/config-paths.js +127 -2
- package/lib/utils/configuration-env-resolver.js +179 -0
- package/lib/utils/credential-display.js +83 -0
- package/lib/utils/credential-secrets-env.js +357 -0
- package/lib/utils/dataplane-pipeline-warning.js +28 -0
- package/lib/utils/deployment-validation-helpers.js +4 -4
- package/lib/utils/dev-ca-install.js +139 -0
- package/lib/utils/dev-cert-helper.js +122 -0
- package/lib/utils/device-code-helpers.js +224 -0
- package/lib/utils/device-code.js +37 -336
- package/lib/utils/docker-build.js +40 -8
- package/lib/utils/env-copy.js +103 -13
- package/lib/utils/env-map.js +35 -5
- package/lib/utils/env-template.js +6 -5
- package/lib/utils/error-formatters/http-status-errors.js +20 -2
- package/lib/utils/error-formatters/permission-errors.js +0 -1
- package/lib/utils/error-formatters/validation-errors.js +0 -1
- package/lib/utils/external-readme.js +56 -29
- package/lib/utils/external-system-display.js +59 -1
- package/lib/utils/external-system-test-helpers.js +21 -8
- package/lib/utils/external-system-validators.js +3 -0
- package/lib/utils/file-upload.js +20 -50
- package/lib/utils/help-builder.js +16 -2
- package/lib/utils/infra-status.js +80 -45
- package/lib/utils/local-secrets.js +7 -52
- package/lib/utils/mutagen-install.js +195 -0
- package/lib/utils/mutagen.js +146 -0
- package/lib/utils/paths.js +128 -37
- package/lib/utils/port-resolver.js +28 -16
- package/lib/utils/remote-dev-auth.js +38 -0
- package/lib/utils/remote-docker-env.js +43 -0
- package/lib/utils/remote-secrets-loader.js +60 -0
- package/lib/utils/secrets-canonical.js +93 -0
- package/lib/utils/secrets-generator.js +114 -6
- package/lib/utils/secrets-helpers.js +108 -114
- package/lib/utils/secrets-path.js +2 -2
- package/lib/utils/secrets-utils.js +52 -1
- package/lib/utils/secrets-validation.js +84 -0
- package/lib/utils/ssh-key-helper.js +116 -0
- package/lib/utils/test-log-writer.js +56 -0
- package/lib/utils/token-manager-messages.js +90 -0
- package/lib/utils/token-manager.js +29 -36
- package/lib/utils/variable-transformer.js +3 -3
- package/lib/validation/env-template-auth.js +157 -0
- package/lib/validation/env-template-kv.js +41 -0
- package/lib/validation/external-manifest-validator.js +25 -0
- package/lib/validation/external-system-auth-rules.js +86 -0
- package/lib/validation/validate-batch.js +149 -0
- package/lib/validation/validate-datasource-keys-api.js +33 -0
- package/lib/validation/validate-display.js +94 -16
- package/lib/validation/validate.js +25 -12
- package/lib/validation/validator.js +72 -9
- package/lib/validation/wizard-datasource-validation.js +50 -0
- package/package.json +8 -3
- package/scripts/install-local.js +34 -15
- package/templates/README.md +0 -1
- package/templates/applications/README.md.hbs +4 -4
- package/templates/applications/dataplane/application.yaml +6 -5
- package/templates/applications/dataplane/env.template +15 -10
- package/templates/applications/dataplane/rbac.yaml +2 -2
- package/templates/applications/keycloak/env.template +2 -0
- package/templates/applications/miso-controller/application.yaml +1 -0
- package/templates/applications/miso-controller/env.template +12 -10
- package/templates/external-system/README.md.hbs +65 -25
- package/templates/external-system/deploy.js.hbs +4 -2
- package/templates/external-system/external-datasource.yaml.hbs +217 -0
- package/templates/external-system/external-system.json.hbs +1 -18
- package/templates/infra/compose.yaml.hbs +6 -0
- package/templates/python/docker-compose.hbs +49 -23
- package/templates/typescript/docker-compose.hbs +48 -22
- package/integration/hubspot/application.yaml +0 -37
package/lib/core/secrets.js
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* @author AI Fabrix Team
|
|
9
9
|
* @version 2.0.0
|
|
10
10
|
*/
|
|
11
|
-
|
|
11
|
+
/* eslint-disable max-lines -- Central module; env-only resolve (plan 75) added required options; extract to env-merge would touch multiple callers. */
|
|
12
12
|
const fs = require('fs');
|
|
13
13
|
const path = require('path');
|
|
14
14
|
const logger = require('../utils/logger');
|
|
@@ -27,20 +27,27 @@ const {
|
|
|
27
27
|
ensureNonEmptySecrets,
|
|
28
28
|
validateSecrets
|
|
29
29
|
} = require('../utils/secrets-helpers');
|
|
30
|
-
const { processEnvVariables } = require('../utils/env-copy');
|
|
31
30
|
const { buildEnvVarMap } = require('../utils/env-map');
|
|
32
31
|
const { resolveServicePortsInEnvContent } = require('../utils/secrets-url');
|
|
33
|
-
const {
|
|
32
|
+
const {
|
|
33
|
+
updatePortForDocker,
|
|
34
|
+
getBaseDockerEnv,
|
|
35
|
+
applyDockerEnvOverride,
|
|
36
|
+
getContainerPortFromDockerEnv
|
|
37
|
+
} = require('./secrets-docker-env');
|
|
38
|
+
const { getContainerPortFromPath } = require('../utils/port-resolver');
|
|
34
39
|
const {
|
|
35
40
|
generateMissingSecrets,
|
|
36
41
|
createDefaultSecrets
|
|
37
42
|
} = require('../utils/secrets-generator');
|
|
43
|
+
const secretsEnsure = require('./secrets-ensure');
|
|
38
44
|
const {
|
|
39
45
|
resolveSecretsPath,
|
|
40
46
|
getActualSecretsPath
|
|
41
47
|
} = require('../utils/secrets-path');
|
|
42
48
|
const {
|
|
43
49
|
loadUserSecrets,
|
|
50
|
+
loadPrimaryUserSecrets,
|
|
44
51
|
loadDefaultSecrets
|
|
45
52
|
} = require('../utils/secrets-utils');
|
|
46
53
|
const { decryptSecret, isEncrypted } = require('../utils/secrets-encryption');
|
|
@@ -162,12 +169,18 @@ function mergeUserWithConfigFile(userSecrets, resolvedConfigPath) {
|
|
|
162
169
|
}
|
|
163
170
|
|
|
164
171
|
/**
|
|
165
|
-
* Loads config secrets path, merges with user secrets (user
|
|
172
|
+
* Loads config secrets path, merges with user secrets (user/master wins, public fills missing).
|
|
173
|
+
* User/master = primary home (AIFABRIX_HOME or ~/.aifabrix) secrets.local.yaml.
|
|
174
|
+
* Public = aifabrix-secrets path from config. Used by loadSecrets cascade.
|
|
175
|
+
* When aifabrix-secrets is an http(s) URL, fetches shared secrets from API (never persisted to disk).
|
|
176
|
+
*
|
|
166
177
|
* @async
|
|
167
178
|
* @returns {Promise<Object|null>} Merged secrets object or null
|
|
168
179
|
*/
|
|
169
180
|
async function loadMergedConfigAndUserSecrets() {
|
|
170
|
-
const
|
|
181
|
+
const { loadRemoteSharedSecrets, mergeUserWithRemoteSecrets } = require('../utils/remote-secrets-loader');
|
|
182
|
+
const { isRemoteSecretsUrl } = require('../utils/remote-dev-auth');
|
|
183
|
+
const userSecrets = loadPrimaryUserSecrets();
|
|
171
184
|
const hasKeys = (obj) => obj && Object.keys(obj).length > 0;
|
|
172
185
|
const userOrNull = () => (hasKeys(userSecrets) ? userSecrets : null);
|
|
173
186
|
try {
|
|
@@ -175,6 +188,11 @@ async function loadMergedConfigAndUserSecrets() {
|
|
|
175
188
|
if (!configSecretsPath) {
|
|
176
189
|
return userOrNull();
|
|
177
190
|
}
|
|
191
|
+
if (isRemoteSecretsUrl(configSecretsPath)) {
|
|
192
|
+
const remoteSecrets = await loadRemoteSharedSecrets();
|
|
193
|
+
const merged = mergeUserWithRemoteSecrets(userSecrets, remoteSecrets);
|
|
194
|
+
return hasKeys(merged) ? merged : userOrNull();
|
|
195
|
+
}
|
|
178
196
|
const resolvedConfigPath = path.isAbsolute(configSecretsPath)
|
|
179
197
|
? configSecretsPath
|
|
180
198
|
: path.resolve(process.cwd(), configSecretsPath);
|
|
@@ -188,6 +206,38 @@ async function loadMergedConfigAndUserSecrets() {
|
|
|
188
206
|
}
|
|
189
207
|
}
|
|
190
208
|
|
|
209
|
+
/**
|
|
210
|
+
* Loads merged secrets using config/user cascade, builder file merge, and default fallback.
|
|
211
|
+
* @async
|
|
212
|
+
* @returns {Promise<Object>} Merged secrets object (not decrypted)
|
|
213
|
+
*/
|
|
214
|
+
async function loadSecretsWithFallbacks() {
|
|
215
|
+
let merged = await loadMergedConfigAndUserSecrets();
|
|
216
|
+
if (!merged || Object.keys(merged).length === 0) {
|
|
217
|
+
merged = loadPrimaryUserSecrets();
|
|
218
|
+
if (Object.keys(merged).length === 0) {
|
|
219
|
+
merged = loadUserSecrets();
|
|
220
|
+
}
|
|
221
|
+
merged = await applyCanonicalSecretsOverride(merged);
|
|
222
|
+
}
|
|
223
|
+
try {
|
|
224
|
+
const projectRoot = pathsUtil.getProjectRoot();
|
|
225
|
+
if (projectRoot) {
|
|
226
|
+
const builderPath = path.join(projectRoot, 'builder', 'secrets.local.yaml');
|
|
227
|
+
if (fs.existsSync(builderPath)) {
|
|
228
|
+
const builderSecrets = mergeUserWithConfigFile(merged || {}, builderPath);
|
|
229
|
+
if (builderSecrets) merged = builderSecrets;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
} catch {
|
|
233
|
+
// Ignore (e.g. no project root or read error)
|
|
234
|
+
}
|
|
235
|
+
if (Object.keys(merged).length === 0) {
|
|
236
|
+
merged = loadDefaultSecrets();
|
|
237
|
+
}
|
|
238
|
+
return merged;
|
|
239
|
+
}
|
|
240
|
+
|
|
191
241
|
async function loadSecrets(secretsPath, _appName) {
|
|
192
242
|
if (secretsPath) {
|
|
193
243
|
const resolvedPath = resolveSecretsPath(secretsPath);
|
|
@@ -200,15 +250,7 @@ async function loadSecrets(secretsPath, _appName) {
|
|
|
200
250
|
}
|
|
201
251
|
return await decryptSecretsObject(explicitSecrets);
|
|
202
252
|
}
|
|
203
|
-
|
|
204
|
-
let mergedSecrets = await loadMergedConfigAndUserSecrets();
|
|
205
|
-
if (!mergedSecrets || Object.keys(mergedSecrets).length === 0) {
|
|
206
|
-
mergedSecrets = loadUserSecrets();
|
|
207
|
-
mergedSecrets = await applyCanonicalSecretsOverride(mergedSecrets);
|
|
208
|
-
}
|
|
209
|
-
if (Object.keys(mergedSecrets).length === 0) {
|
|
210
|
-
mergedSecrets = loadDefaultSecrets();
|
|
211
|
-
}
|
|
253
|
+
const mergedSecrets = await loadSecretsWithFallbacks();
|
|
212
254
|
ensureNonEmptySecrets(mergedSecrets);
|
|
213
255
|
return await decryptSecretsObject(mergedSecrets);
|
|
214
256
|
}
|
|
@@ -259,44 +301,57 @@ async function resolveKvReferences(envTemplate, secrets, environment = 'local',
|
|
|
259
301
|
return replaceKvInContent(resolved, secrets, envVars);
|
|
260
302
|
}
|
|
261
303
|
|
|
262
|
-
/**
|
|
304
|
+
/** Docker env transformations: ports, infra endpoints, PORT. */
|
|
305
|
+
async function applyDockerTransformations(resolved, variablesPath) {
|
|
306
|
+
resolved = await resolveServicePortsInEnvContent(resolved, 'docker');
|
|
307
|
+
resolved = await rewriteInfraEndpoints(resolved, 'docker');
|
|
308
|
+
const { getEnvHosts, getServiceHost, getServicePort, getLocalhostOverride } = require('../utils/env-endpoints');
|
|
309
|
+
const hosts = await getEnvHosts('docker');
|
|
310
|
+
const localhostOverride = getLocalhostOverride('docker');
|
|
311
|
+
const redisHost = getServiceHost(hosts.REDIS_HOST, 'docker', 'redis', localhostOverride);
|
|
312
|
+
const redisPort = await getServicePort('REDIS_PORT', 'redis', hosts, 'docker', null);
|
|
313
|
+
const dbHost = getServiceHost(hosts.DB_HOST, 'docker', 'postgres', localhostOverride);
|
|
314
|
+
const dbPort = await getServicePort('DB_PORT', 'postgres', hosts, 'docker', null);
|
|
315
|
+
let dockerEnv = await getBaseDockerEnv();
|
|
316
|
+
dockerEnv = applyDockerEnvOverride(dockerEnv);
|
|
317
|
+
const containerPort = getContainerPortFromPath(variablesPath) ?? getContainerPortFromDockerEnv(dockerEnv) ?? 3000;
|
|
318
|
+
const envVars = await buildEnvVarMap('docker', null, null, { appPort: containerPort });
|
|
319
|
+
envVars.REDIS_HOST = redisHost;
|
|
320
|
+
envVars.REDIS_PORT = String(redisPort);
|
|
321
|
+
envVars.DB_HOST = dbHost;
|
|
322
|
+
envVars.DB_PORT = String(dbPort);
|
|
323
|
+
envVars.PORT = String(containerPort);
|
|
324
|
+
resolved = interpolateEnvVars(resolved, envVars);
|
|
325
|
+
return updatePortForDocker(resolved, variablesPath);
|
|
326
|
+
}
|
|
327
|
+
/** Environment-specific transformations to resolved content. */
|
|
263
328
|
async function applyEnvironmentTransformations(resolved, environment, variablesPath) {
|
|
264
|
-
if (environment === 'docker')
|
|
265
|
-
|
|
266
|
-
resolved = await rewriteInfraEndpoints(resolved, 'docker');
|
|
267
|
-
const { getEnvHosts, getServiceHost, getServicePort, getLocalhostOverride } = require('../utils/env-endpoints');
|
|
268
|
-
const hosts = await getEnvHosts('docker');
|
|
269
|
-
const localhostOverride = getLocalhostOverride('docker');
|
|
270
|
-
const redisHost = getServiceHost(hosts.REDIS_HOST, 'docker', 'redis', localhostOverride);
|
|
271
|
-
const redisPort = await getServicePort('REDIS_PORT', 'redis', hosts, 'docker', null);
|
|
272
|
-
const dbHost = getServiceHost(hosts.DB_HOST, 'docker', 'postgres', localhostOverride);
|
|
273
|
-
const dbPort = await getServicePort('DB_PORT', 'postgres', hosts, 'docker', null);
|
|
274
|
-
const envVars = await buildEnvVarMap('docker');
|
|
275
|
-
envVars.REDIS_HOST = redisHost;
|
|
276
|
-
envVars.REDIS_PORT = String(redisPort);
|
|
277
|
-
envVars.DB_HOST = dbHost;
|
|
278
|
-
envVars.DB_PORT = String(dbPort);
|
|
279
|
-
resolved = interpolateEnvVars(resolved, envVars);
|
|
280
|
-
resolved = await updatePortForDocker(resolved, variablesPath);
|
|
281
|
-
} else if (environment === 'local') {
|
|
282
|
-
// adjustLocalEnvPortsInContent handles both PORT and infra endpoints
|
|
283
|
-
resolved = await adjustLocalEnvPortsInContent(resolved, variablesPath);
|
|
284
|
-
}
|
|
329
|
+
if (environment === 'docker') return applyDockerTransformations(resolved, variablesPath);
|
|
330
|
+
if (environment === 'local') return adjustLocalEnvPortsInContent(resolved, variablesPath);
|
|
285
331
|
return resolved;
|
|
286
332
|
}
|
|
287
333
|
|
|
288
|
-
/**
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
334
|
+
/**
|
|
335
|
+
* Generate .env content from template and secrets (no disk write).
|
|
336
|
+
* When options.envOnly is true, variablesPath is null (no application config).
|
|
337
|
+
*
|
|
338
|
+
* @param {string} appName - Application name
|
|
339
|
+
* @param {string} [secretsPath] - Path to secrets file (optional)
|
|
340
|
+
* @param {string} [environment='local'] - Environment context
|
|
341
|
+
* @param {boolean} [force=false] - Generate missing secret keys
|
|
342
|
+
* @param {Object} [options] - Optional: appPath, envOnly (env-only mode uses only env.template)
|
|
343
|
+
* @returns {Promise<string>} Resolved env content
|
|
344
|
+
*/
|
|
345
|
+
async function generateEnvContent(appName, secretsPath, environment = 'local', force = false, options = {}) {
|
|
346
|
+
const appPath = (options && options.appPath) || pathsUtil.getBuilderPath(appName);
|
|
347
|
+
const templatePath = path.join(appPath, 'env.template');
|
|
348
|
+
const variablesPath = (options && options.envOnly) ? null : resolveApplicationConfigPath(appPath);
|
|
293
349
|
const template = loadEnvTemplate(templatePath);
|
|
294
350
|
const secretsPaths = await getActualSecretsPath(secretsPath, appName);
|
|
295
351
|
if (force) {
|
|
296
|
-
const
|
|
297
|
-
await
|
|
352
|
+
const preferredPath = secretsPath ? resolveSecretsPath(secretsPath) : secretsPaths.userPath;
|
|
353
|
+
await secretsEnsure.ensureSecretsFromEnvTemplate(templatePath, { preferredFilePath: preferredPath });
|
|
298
354
|
}
|
|
299
|
-
|
|
300
355
|
const secrets = await loadSecrets(secretsPath, appName);
|
|
301
356
|
let resolved = await resolveKvReferences(template, secrets, environment, secretsPaths, appName);
|
|
302
357
|
resolved = await applyEnvironmentTransformations(resolved, environment, variablesPath);
|
|
@@ -371,73 +426,104 @@ function mergeEnvContentPreservingExisting(newContent, existingMap) {
|
|
|
371
426
|
}
|
|
372
427
|
|
|
373
428
|
/**
|
|
374
|
-
*
|
|
375
|
-
*
|
|
376
|
-
*
|
|
429
|
+
* Merges a key-value map into existing .env file content, preserving comments and blank lines.
|
|
430
|
+
* For each KEY=value line in existing content, replaces value with newMap[KEY] when the key exists
|
|
431
|
+
* in newMap. Appends any keys from newMap that did not appear in the file.
|
|
377
432
|
*
|
|
433
|
+
* @function mergeEnvMapIntoContent
|
|
434
|
+
* @param {string} existingContent - Full existing .env file content
|
|
435
|
+
* @param {Object.<string, string>} newMap - New key-to-value map (e.g. from resolved or run env)
|
|
436
|
+
* @returns {string} Merged content with comments preserved
|
|
437
|
+
*/
|
|
438
|
+
function mergeEnvMapIntoContent(existingContent, newMap) {
|
|
439
|
+
if (!newMap || Object.keys(newMap).length === 0) {
|
|
440
|
+
return typeof existingContent === 'string' ? existingContent : '';
|
|
441
|
+
}
|
|
442
|
+
const lines = (existingContent || '').split(/\r?\n/);
|
|
443
|
+
const seen = new Set();
|
|
444
|
+
const out = [];
|
|
445
|
+
for (const line of lines) {
|
|
446
|
+
const trimmed = line.trim();
|
|
447
|
+
if (!trimmed || trimmed.startsWith('#')) {
|
|
448
|
+
out.push(line);
|
|
449
|
+
continue;
|
|
450
|
+
}
|
|
451
|
+
const eq = trimmed.indexOf('=');
|
|
452
|
+
if (eq > 0) {
|
|
453
|
+
const key = trimmed.substring(0, eq).trim();
|
|
454
|
+
seen.add(key);
|
|
455
|
+
out.push(Object.prototype.hasOwnProperty.call(newMap, key) ? `${key}=${newMap[key]}` : line);
|
|
456
|
+
continue;
|
|
457
|
+
}
|
|
458
|
+
out.push(line);
|
|
459
|
+
}
|
|
460
|
+
for (const key of Object.keys(newMap)) {
|
|
461
|
+
if (!seen.has(key)) out.push(`${key}=${newMap[key]}`);
|
|
462
|
+
}
|
|
463
|
+
return out.join('\n');
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* Resolves content to write for .env: merges with existing file when present.
|
|
468
|
+
* @param {string} resolved - Newly generated content
|
|
469
|
+
* @param {string} pathToPreserve - Path to existing .env to merge from (or null)
|
|
470
|
+
* @returns {string} Content to write
|
|
471
|
+
*/
|
|
472
|
+
function resolveEnvContentToWrite(resolved, pathToPreserve) {
|
|
473
|
+
if (!pathToPreserve || !fs.existsSync(pathToPreserve)) return resolved;
|
|
474
|
+
const existingContent = fs.readFileSync(pathToPreserve, 'utf8');
|
|
475
|
+
const existingMap = parseEnvContentToMap(existingContent);
|
|
476
|
+
return mergeEnvContentPreservingExisting(resolved, existingMap);
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/**
|
|
480
|
+
* Generates and writes .env file. Newly resolved values win over existing .env; extra vars in existing .env are kept.
|
|
481
|
+
* When options.envOnly is true, only env.template is used; .env is written to options.appPath.
|
|
378
482
|
* @async
|
|
379
483
|
* @function generateEnvFile
|
|
380
484
|
* @param {string} appName - Name of the application
|
|
381
485
|
* @param {string} [secretsPath] - Path to secrets file (optional)
|
|
382
486
|
* @param {string} [environment='local'] - Environment context ('local' or 'docker')
|
|
383
487
|
* @param {boolean} [force=false] - Generate missing secret keys in secrets file
|
|
384
|
-
* @param {
|
|
385
|
-
* @param {string} [preserveFromPath=null] - Path to existing .env to preserve values from (defaults to builder .env)
|
|
488
|
+
* @param {Object} [options] - Optional: appPath, envOnly, skipOutputPath, preserveFromPath
|
|
386
489
|
* @returns {Promise<string>} Path to generated .env file
|
|
387
|
-
* @throws {Error} If generation fails
|
|
388
|
-
*
|
|
389
|
-
* @example
|
|
390
|
-
* const envPath = await generateEnvFile('myapp', undefined, 'docker');
|
|
391
|
-
* // When builder/myapp/.env already exists, existing values are preserved
|
|
392
490
|
*/
|
|
393
|
-
async function generateEnvFile(appName, secretsPath, environment = 'local', force = false,
|
|
394
|
-
const
|
|
395
|
-
const
|
|
396
|
-
const
|
|
397
|
-
|
|
398
|
-
const
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
toWrite = mergeEnvContentPreservingExisting(resolved, existingMap);
|
|
491
|
+
async function generateEnvFile(appName, secretsPath, environment = 'local', force = false, options = {}) {
|
|
492
|
+
const opts = options && typeof options === 'object' ? options : {};
|
|
493
|
+
const appPath = opts.appPath || pathsUtil.getBuilderPath(appName);
|
|
494
|
+
const envOnly = !!opts.envOnly;
|
|
495
|
+
const variablesPath = envOnly ? null : resolveApplicationConfigPath(appPath);
|
|
496
|
+
const envPath = path.join(appPath, '.env');
|
|
497
|
+
|
|
498
|
+
if (envOnly) {
|
|
499
|
+
const templatePath = path.join(appPath, 'env.template');
|
|
500
|
+
if (!fs.existsSync(templatePath)) {
|
|
501
|
+
throw new Error(`env.template not found at ${templatePath}. Resolve requires env.template in the app directory.`);
|
|
502
|
+
}
|
|
406
503
|
}
|
|
407
504
|
|
|
505
|
+
const resolved = await generateEnvContent(appName, secretsPath, environment, force, { appPath, envOnly });
|
|
506
|
+
const preservePath = opts.preserveFromPath !== undefined && opts.preserveFromPath !== null ? opts.preserveFromPath : null;
|
|
507
|
+
const pathToPreserve = preservePath !== null ? preservePath : envPath;
|
|
508
|
+
const toWrite = resolveEnvContentToWrite(resolved, pathToPreserve);
|
|
408
509
|
fs.writeFileSync(envPath, toWrite, { mode: 0o600 });
|
|
409
510
|
|
|
410
|
-
|
|
411
|
-
|
|
511
|
+
if (!opts.skipOutputPath) {
|
|
512
|
+
const { processEnvVariables } = require('../utils/env-copy');
|
|
412
513
|
await processEnvVariables(envPath, variablesPath, appName, secretsPath);
|
|
413
514
|
}
|
|
414
515
|
|
|
415
516
|
return envPath;
|
|
416
517
|
}
|
|
417
518
|
|
|
418
|
-
/**
|
|
419
|
-
* Generates admin secrets for infrastructure
|
|
420
|
-
* Creates ~/.aifabrix/admin-secrets.env with Postgres and Redis credentials
|
|
421
|
-
*
|
|
422
|
-
* @async
|
|
423
|
-
* @function generateAdminSecretsEnv
|
|
424
|
-
* @param {string} [secretsPath] - Path to secrets file (optional)
|
|
425
|
-
* @returns {Promise<string>} Path to generated admin-secrets.env file
|
|
426
|
-
* @throws {Error} If generation fails
|
|
427
|
-
*
|
|
428
|
-
* @example
|
|
429
|
-
* const adminEnvPath = await generateAdminSecretsEnv('../../secrets.local.yaml');
|
|
430
|
-
* // Returns: '~/.aifabrix/admin-secrets.env'
|
|
431
|
-
*/
|
|
519
|
+
/** Generates admin secrets for infrastructure (~/.aifabrix/admin-secrets.env). Uses admin123 when no postgres password. */
|
|
432
520
|
async function generateAdminSecretsEnv(secretsPath) {
|
|
433
521
|
let secrets;
|
|
434
522
|
|
|
435
523
|
try {
|
|
436
524
|
secrets = await loadSecrets(secretsPath);
|
|
437
525
|
} catch (error) {
|
|
438
|
-
// If secrets file doesn't exist, create default secrets
|
|
439
526
|
const defaultSecretsPath = secretsPath || path.join(pathsUtil.getAifabrixHome(), 'secrets.yaml');
|
|
440
|
-
|
|
441
527
|
if (!fs.existsSync(defaultSecretsPath)) {
|
|
442
528
|
logger.log('Creating default secrets file...');
|
|
443
529
|
await createDefaultSecrets(defaultSecretsPath);
|
|
@@ -446,15 +532,14 @@ async function generateAdminSecretsEnv(secretsPath) {
|
|
|
446
532
|
throw error;
|
|
447
533
|
}
|
|
448
534
|
}
|
|
449
|
-
|
|
450
535
|
const aifabrixDir = pathsUtil.getAifabrixHome();
|
|
451
536
|
const adminEnvPath = path.join(aifabrixDir, 'admin-secrets.env');
|
|
452
|
-
|
|
453
537
|
if (!fs.existsSync(aifabrixDir)) {
|
|
454
538
|
fs.mkdirSync(aifabrixDir, { recursive: true, mode: 0o700 });
|
|
455
539
|
}
|
|
456
540
|
|
|
457
|
-
const
|
|
541
|
+
const raw = secrets['postgres-passwordKeyVault'];
|
|
542
|
+
const postgresPassword = (raw && String(raw).trim()) || 'admin123';
|
|
458
543
|
|
|
459
544
|
const adminSecrets = `# Infrastructure Admin Credentials
|
|
460
545
|
POSTGRES_PASSWORD=${postgresPassword}
|
|
@@ -477,5 +562,7 @@ module.exports = {
|
|
|
477
562
|
generateAdminSecretsEnv,
|
|
478
563
|
validateSecrets,
|
|
479
564
|
createDefaultSecrets,
|
|
480
|
-
getCanonicalSecretName
|
|
565
|
+
getCanonicalSecretName,
|
|
566
|
+
parseEnvContentToMap,
|
|
567
|
+
mergeEnvMapIntoContent
|
|
481
568
|
};
|
package/lib/datasource/deploy.js
CHANGED
|
@@ -16,6 +16,11 @@ const { getEnvironmentApplication } = require('../api/environments.api');
|
|
|
16
16
|
const { publishDatasourceViaPipeline } = require('../api/pipeline.api');
|
|
17
17
|
const { formatApiError } = require('../utils/api-error-handler');
|
|
18
18
|
const logger = require('../utils/logger');
|
|
19
|
+
const { logDataplanePipelineWarning } = require('../utils/dataplane-pipeline-warning');
|
|
20
|
+
const {
|
|
21
|
+
buildResolvedEnvMapForIntegration,
|
|
22
|
+
resolveConfigurationValues
|
|
23
|
+
} = require('../utils/configuration-env-resolver');
|
|
19
24
|
const { validateDatasourceFile } = require('./validate');
|
|
20
25
|
|
|
21
26
|
/**
|
|
@@ -50,7 +55,7 @@ async function getDataplaneUrl(controllerUrl, appKey, environment, authConfig) {
|
|
|
50
55
|
if (!dataplaneUrl) {
|
|
51
56
|
const appType = application.configuration?.type || application.type;
|
|
52
57
|
if (appType === 'external') {
|
|
53
|
-
throw new Error('Dataplane URL not found for external system
|
|
58
|
+
throw new Error('Dataplane URL not found for external system in application configuration.');
|
|
54
59
|
}
|
|
55
60
|
throw new Error('Dataplane URL not found in application configuration');
|
|
56
61
|
}
|
|
@@ -107,8 +112,6 @@ async function validateAndLoadDatasourceFile(filePath) {
|
|
|
107
112
|
* @param {string} controllerUrl - Controller URL
|
|
108
113
|
* @param {string} environment - Environment key
|
|
109
114
|
* @param {string} appKey - Application key
|
|
110
|
-
* @param {Object} [options] - Options
|
|
111
|
-
* @param {string} [options.dataplane] - Dataplane URL override
|
|
112
115
|
* @returns {Promise<Object>} Object with authConfig and dataplaneUrl
|
|
113
116
|
*/
|
|
114
117
|
async function setupDeploymentAuth(controllerUrl, environment, appKey) {
|
|
@@ -154,6 +157,7 @@ async function setupDeploymentAuth(controllerUrl, environment, appKey) {
|
|
|
154
157
|
async function publishDatasourceToDataplane(dataplaneUrl, systemKey, authConfig, datasourceConfig) {
|
|
155
158
|
requireBearerForDataplanePipeline(authConfig);
|
|
156
159
|
logger.log(chalk.blue('\nš Publishing datasource to dataplane...'));
|
|
160
|
+
logDataplanePipelineWarning();
|
|
157
161
|
|
|
158
162
|
const publishResponse = await publishDatasourceViaPipeline(dataplaneUrl, systemKey, authConfig, datasourceConfig);
|
|
159
163
|
|
|
@@ -228,6 +232,11 @@ async function deployDatasource(appKey, filePath, _options) {
|
|
|
228
232
|
throw new Error('systemKey is required in datasource configuration');
|
|
229
233
|
}
|
|
230
234
|
|
|
235
|
+
if (Array.isArray(datasourceConfig.configuration) && datasourceConfig.configuration.length > 0) {
|
|
236
|
+
const { envMap, secrets } = await buildResolvedEnvMapForIntegration(systemKey);
|
|
237
|
+
resolveConfigurationValues(datasourceConfig.configuration, envMap, secrets, systemKey);
|
|
238
|
+
}
|
|
239
|
+
|
|
231
240
|
// Setup authentication and get dataplane URL
|
|
232
241
|
const { authConfig, dataplaneUrl } = await setupDeploymentAuth(controllerUrl, environment, appKey);
|
|
233
242
|
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Field Reference Validator for External Datasource
|
|
3
|
+
*
|
|
4
|
+
* Validates that field names used in indexing (embedding, uniqueKey),
|
|
5
|
+
* validation.repeatingValues[].field, and quality.rejectIf[].field exist in
|
|
6
|
+
* fieldMappings.attributes. Aligns with dataplane invalid_reference semantics.
|
|
7
|
+
*
|
|
8
|
+
* @fileoverview Offline field reference validation for AI Fabrix Builder
|
|
9
|
+
* @author AI Fabrix Team
|
|
10
|
+
* @version 2.0.0
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Validates that all field references in indexing, validation, and quality
|
|
15
|
+
* exist in fieldMappings.attributes. When fieldMappings.attributes is missing
|
|
16
|
+
* or empty, returns no errors (skip check, matching dataplane behavior).
|
|
17
|
+
*
|
|
18
|
+
* @function validateFieldReferences
|
|
19
|
+
* @param {Object} parsed - Parsed datasource object (after JSON parse)
|
|
20
|
+
* @returns {string[]} Array of error messages; empty if no invalid references
|
|
21
|
+
*
|
|
22
|
+
* @example
|
|
23
|
+
* const errors = validateFieldReferences(parsed);
|
|
24
|
+
* if (errors.length > 0) {
|
|
25
|
+
* errors.forEach(e => console.error(e));
|
|
26
|
+
* }
|
|
27
|
+
*/
|
|
28
|
+
function validateFieldReferences(parsed) {
|
|
29
|
+
const errors = [];
|
|
30
|
+
const normalizedAttributes = Object.keys(
|
|
31
|
+
parsed?.fieldMappings?.attributes ?? {}
|
|
32
|
+
);
|
|
33
|
+
|
|
34
|
+
if (normalizedAttributes.length === 0) {
|
|
35
|
+
return [];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// indexing.embedding: array of field names
|
|
39
|
+
const embedding = parsed?.indexing?.embedding;
|
|
40
|
+
if (Array.isArray(embedding)) {
|
|
41
|
+
embedding.forEach((field, i) => {
|
|
42
|
+
if (typeof field === 'string' && !normalizedAttributes.includes(field)) {
|
|
43
|
+
errors.push(
|
|
44
|
+
`indexing.embedding[${i}]: field '${field}' does not exist in fieldMappings.attributes`
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// indexing.uniqueKey: single field name
|
|
51
|
+
const uniqueKey = parsed?.indexing?.uniqueKey;
|
|
52
|
+
if (typeof uniqueKey === 'string' && uniqueKey !== '') {
|
|
53
|
+
if (!normalizedAttributes.includes(uniqueKey)) {
|
|
54
|
+
errors.push(
|
|
55
|
+
`indexing.uniqueKey: field '${uniqueKey}' does not exist in fieldMappings.attributes`
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// validation.repeatingValues[].field
|
|
61
|
+
const repeatingValues = parsed?.validation?.repeatingValues;
|
|
62
|
+
if (Array.isArray(repeatingValues)) {
|
|
63
|
+
repeatingValues.forEach((rule, index) => {
|
|
64
|
+
const field = rule?.field;
|
|
65
|
+
if (typeof field === 'string' && !normalizedAttributes.includes(field)) {
|
|
66
|
+
errors.push(
|
|
67
|
+
`validation.repeatingValues[${index}].field: field '${field}' does not exist in fieldMappings.attributes`
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// quality.rejectIf[].field
|
|
74
|
+
const rejectIf = parsed?.quality?.rejectIf;
|
|
75
|
+
if (Array.isArray(rejectIf)) {
|
|
76
|
+
rejectIf.forEach((rule, index) => {
|
|
77
|
+
const field = rule?.field;
|
|
78
|
+
if (typeof field === 'string' && !normalizedAttributes.includes(field)) {
|
|
79
|
+
errors.push(
|
|
80
|
+
`quality.rejectIf[${index}].field: field '${field}' does not exist in fieldMappings.attributes`
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return errors;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
module.exports = {
|
|
90
|
+
validateFieldReferences
|
|
91
|
+
};
|