@aifabrix/builder 2.40.0 → 2.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -5
- package/integration/hubspot/test.js +1 -1
- package/jest.config.manual.js +29 -0
- package/lib/api/credential.api.js +40 -0
- package/lib/api/dev.api.js +423 -0
- package/lib/api/types/credential.types.js +23 -0
- package/lib/api/types/dev.types.js +140 -0
- package/lib/app/config.js +21 -0
- package/lib/app/down.js +2 -1
- package/lib/app/index.js +9 -0
- package/lib/app/push.js +36 -12
- package/lib/app/readme.js +1 -3
- package/lib/app/run-env-compose.js +201 -0
- package/lib/app/run-helpers.js +121 -118
- package/lib/app/run.js +148 -28
- package/lib/app/show.js +5 -2
- package/lib/build/index.js +11 -3
- package/lib/cli/setup-app.js +140 -14
- package/lib/cli/setup-auth.js +1 -0
- package/lib/cli/setup-dev.js +180 -17
- package/lib/cli/setup-environment.js +4 -2
- package/lib/cli/setup-external-system.js +71 -21
- package/lib/cli/setup-infra.js +29 -2
- package/lib/cli/setup-secrets.js +52 -5
- package/lib/cli/setup-utility.js +19 -4
- package/lib/commands/app-install.js +172 -0
- package/lib/commands/app-shell.js +75 -0
- package/lib/commands/app-test.js +282 -0
- package/lib/commands/app.js +1 -1
- package/lib/commands/auth-status.js +36 -3
- package/lib/commands/dev-cli-handlers.js +141 -0
- package/lib/commands/dev-down.js +114 -0
- package/lib/commands/dev-init.js +309 -0
- package/lib/commands/secrets-list.js +118 -0
- package/lib/commands/secrets-remove.js +97 -0
- package/lib/commands/secrets-set.js +30 -17
- package/lib/commands/secrets-validate.js +50 -0
- package/lib/commands/up-dataplane.js +2 -2
- package/lib/commands/up-miso.js +0 -25
- package/lib/commands/upload.js +26 -1
- package/lib/core/admin-secrets.js +96 -0
- package/lib/core/secrets-ensure.js +378 -0
- package/lib/core/secrets-env-write.js +157 -0
- package/lib/core/secrets.js +147 -81
- package/lib/datasource/field-reference-validator.js +91 -0
- package/lib/datasource/validate.js +21 -3
- package/lib/deployment/environment-config.js +137 -0
- package/lib/deployment/environment.js +21 -98
- package/lib/deployment/push.js +32 -2
- package/lib/external-system/download.js +7 -0
- package/lib/external-system/test-auth.js +7 -3
- package/lib/external-system/test.js +5 -1
- package/lib/generator/index.js +174 -25
- package/lib/generator/wizard.js +13 -1
- package/lib/infrastructure/helpers.js +103 -20
- package/lib/infrastructure/index.js +88 -10
- package/lib/infrastructure/services.js +70 -15
- package/lib/schema/application-schema.json +24 -3
- package/lib/schema/external-system.schema.json +435 -413
- package/lib/utils/api.js +3 -3
- package/lib/utils/app-register-auth.js +25 -3
- package/lib/utils/cli-utils.js +20 -0
- package/lib/utils/compose-generator.js +76 -75
- package/lib/utils/compose-handlebars-helpers.js +43 -0
- package/lib/utils/compose-vector-helper.js +18 -0
- package/lib/utils/config-paths.js +127 -2
- package/lib/utils/credential-secrets-env.js +267 -0
- package/lib/utils/dev-cert-helper.js +122 -0
- package/lib/utils/device-code-helpers.js +224 -0
- package/lib/utils/device-code.js +37 -336
- package/lib/utils/docker-build.js +40 -8
- package/lib/utils/env-copy.js +83 -13
- package/lib/utils/env-map.js +35 -5
- package/lib/utils/env-template.js +6 -5
- package/lib/utils/error-formatters/http-status-errors.js +20 -1
- package/lib/utils/help-builder.js +15 -2
- package/lib/utils/infra-status.js +30 -1
- package/lib/utils/local-secrets.js +7 -52
- package/lib/utils/mutagen-install.js +195 -0
- package/lib/utils/mutagen.js +146 -0
- package/lib/utils/paths.js +49 -33
- package/lib/utils/port-resolver.js +28 -16
- package/lib/utils/remote-dev-auth.js +38 -0
- package/lib/utils/remote-docker-env.js +43 -0
- package/lib/utils/remote-secrets-loader.js +60 -0
- package/lib/utils/secrets-generator.js +94 -6
- package/lib/utils/secrets-helpers.js +33 -25
- package/lib/utils/secrets-path.js +2 -2
- package/lib/utils/secrets-utils.js +52 -1
- package/lib/utils/secrets-validation.js +84 -0
- package/lib/utils/ssh-key-helper.js +116 -0
- package/lib/utils/token-manager-messages.js +90 -0
- package/lib/utils/token-manager.js +5 -4
- package/lib/utils/variable-transformer.js +3 -3
- package/lib/validation/validate.js +1 -1
- package/lib/validation/validator.js +65 -0
- package/package.json +4 -2
- package/scripts/install-local.js +34 -15
- package/templates/README.md +0 -1
- package/templates/applications/README.md.hbs +4 -4
- package/templates/applications/dataplane/application.yaml +5 -4
- package/templates/applications/dataplane/env.template +12 -7
- package/templates/applications/keycloak/env.template +2 -0
- package/templates/applications/miso-controller/application.yaml +1 -0
- package/templates/applications/miso-controller/env.template +11 -9
- package/templates/external-system/external-system.json.hbs +1 -16
- package/templates/python/docker-compose.hbs +49 -23
- package/templates/typescript/docker-compose.hbs +48 -22
package/lib/core/secrets.js
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* @author AI Fabrix Team
|
|
9
9
|
* @version 2.0.0
|
|
10
10
|
*/
|
|
11
|
-
|
|
11
|
+
/* eslint-disable max-lines -- Central module; env-only resolve (plan 75) added required options; extract to env-merge would touch multiple callers. */
|
|
12
12
|
const fs = require('fs');
|
|
13
13
|
const path = require('path');
|
|
14
14
|
const logger = require('../utils/logger');
|
|
@@ -27,20 +27,27 @@ const {
|
|
|
27
27
|
ensureNonEmptySecrets,
|
|
28
28
|
validateSecrets
|
|
29
29
|
} = require('../utils/secrets-helpers');
|
|
30
|
-
const { processEnvVariables } = require('../utils/env-copy');
|
|
31
30
|
const { buildEnvVarMap } = require('../utils/env-map');
|
|
32
31
|
const { resolveServicePortsInEnvContent } = require('../utils/secrets-url');
|
|
33
|
-
const {
|
|
32
|
+
const {
|
|
33
|
+
updatePortForDocker,
|
|
34
|
+
getBaseDockerEnv,
|
|
35
|
+
applyDockerEnvOverride,
|
|
36
|
+
getContainerPortFromDockerEnv
|
|
37
|
+
} = require('./secrets-docker-env');
|
|
38
|
+
const { getContainerPortFromPath } = require('../utils/port-resolver');
|
|
34
39
|
const {
|
|
35
40
|
generateMissingSecrets,
|
|
36
41
|
createDefaultSecrets
|
|
37
42
|
} = require('../utils/secrets-generator');
|
|
43
|
+
const secretsEnsure = require('./secrets-ensure');
|
|
38
44
|
const {
|
|
39
45
|
resolveSecretsPath,
|
|
40
46
|
getActualSecretsPath
|
|
41
47
|
} = require('../utils/secrets-path');
|
|
42
48
|
const {
|
|
43
49
|
loadUserSecrets,
|
|
50
|
+
loadPrimaryUserSecrets,
|
|
44
51
|
loadDefaultSecrets
|
|
45
52
|
} = require('../utils/secrets-utils');
|
|
46
53
|
const { decryptSecret, isEncrypted } = require('../utils/secrets-encryption');
|
|
@@ -162,12 +169,18 @@ function mergeUserWithConfigFile(userSecrets, resolvedConfigPath) {
|
|
|
162
169
|
}
|
|
163
170
|
|
|
164
171
|
/**
|
|
165
|
-
* Loads config secrets path, merges with user secrets (user
|
|
172
|
+
* Loads config secrets path, merges with user secrets (user/master wins, public fills missing).
|
|
173
|
+
* User/master = primary home (AIFABRIX_HOME or ~/.aifabrix) secrets.local.yaml.
|
|
174
|
+
* Public = aifabrix-secrets path from config. Used by loadSecrets cascade.
|
|
175
|
+
* When aifabrix-secrets is an http(s) URL, fetches shared secrets from API (never persisted to disk).
|
|
176
|
+
*
|
|
166
177
|
* @async
|
|
167
178
|
* @returns {Promise<Object|null>} Merged secrets object or null
|
|
168
179
|
*/
|
|
169
180
|
async function loadMergedConfigAndUserSecrets() {
|
|
170
|
-
const
|
|
181
|
+
const { loadRemoteSharedSecrets, mergeUserWithRemoteSecrets } = require('../utils/remote-secrets-loader');
|
|
182
|
+
const { isRemoteSecretsUrl } = require('../utils/remote-dev-auth');
|
|
183
|
+
const userSecrets = loadPrimaryUserSecrets();
|
|
171
184
|
const hasKeys = (obj) => obj && Object.keys(obj).length > 0;
|
|
172
185
|
const userOrNull = () => (hasKeys(userSecrets) ? userSecrets : null);
|
|
173
186
|
try {
|
|
@@ -175,6 +188,11 @@ async function loadMergedConfigAndUserSecrets() {
|
|
|
175
188
|
if (!configSecretsPath) {
|
|
176
189
|
return userOrNull();
|
|
177
190
|
}
|
|
191
|
+
if (isRemoteSecretsUrl(configSecretsPath)) {
|
|
192
|
+
const remoteSecrets = await loadRemoteSharedSecrets();
|
|
193
|
+
const merged = mergeUserWithRemoteSecrets(userSecrets, remoteSecrets);
|
|
194
|
+
return hasKeys(merged) ? merged : userOrNull();
|
|
195
|
+
}
|
|
178
196
|
const resolvedConfigPath = path.isAbsolute(configSecretsPath)
|
|
179
197
|
? configSecretsPath
|
|
180
198
|
: path.resolve(process.cwd(), configSecretsPath);
|
|
@@ -203,7 +221,10 @@ async function loadSecrets(secretsPath, _appName) {
|
|
|
203
221
|
|
|
204
222
|
let mergedSecrets = await loadMergedConfigAndUserSecrets();
|
|
205
223
|
if (!mergedSecrets || Object.keys(mergedSecrets).length === 0) {
|
|
206
|
-
mergedSecrets =
|
|
224
|
+
mergedSecrets = loadPrimaryUserSecrets();
|
|
225
|
+
if (Object.keys(mergedSecrets).length === 0) {
|
|
226
|
+
mergedSecrets = loadUserSecrets();
|
|
227
|
+
}
|
|
207
228
|
mergedSecrets = await applyCanonicalSecretsOverride(mergedSecrets);
|
|
208
229
|
}
|
|
209
230
|
if (Object.keys(mergedSecrets).length === 0) {
|
|
@@ -259,44 +280,57 @@ async function resolveKvReferences(envTemplate, secrets, environment = 'local',
|
|
|
259
280
|
return replaceKvInContent(resolved, secrets, envVars);
|
|
260
281
|
}
|
|
261
282
|
|
|
262
|
-
/**
|
|
283
|
+
/** Docker env transformations: ports, infra endpoints, PORT. */
|
|
284
|
+
async function applyDockerTransformations(resolved, variablesPath) {
|
|
285
|
+
resolved = await resolveServicePortsInEnvContent(resolved, 'docker');
|
|
286
|
+
resolved = await rewriteInfraEndpoints(resolved, 'docker');
|
|
287
|
+
const { getEnvHosts, getServiceHost, getServicePort, getLocalhostOverride } = require('../utils/env-endpoints');
|
|
288
|
+
const hosts = await getEnvHosts('docker');
|
|
289
|
+
const localhostOverride = getLocalhostOverride('docker');
|
|
290
|
+
const redisHost = getServiceHost(hosts.REDIS_HOST, 'docker', 'redis', localhostOverride);
|
|
291
|
+
const redisPort = await getServicePort('REDIS_PORT', 'redis', hosts, 'docker', null);
|
|
292
|
+
const dbHost = getServiceHost(hosts.DB_HOST, 'docker', 'postgres', localhostOverride);
|
|
293
|
+
const dbPort = await getServicePort('DB_PORT', 'postgres', hosts, 'docker', null);
|
|
294
|
+
let dockerEnv = await getBaseDockerEnv();
|
|
295
|
+
dockerEnv = applyDockerEnvOverride(dockerEnv);
|
|
296
|
+
const containerPort = getContainerPortFromPath(variablesPath) ?? getContainerPortFromDockerEnv(dockerEnv) ?? 3000;
|
|
297
|
+
const envVars = await buildEnvVarMap('docker', null, null, { appPort: containerPort });
|
|
298
|
+
envVars.REDIS_HOST = redisHost;
|
|
299
|
+
envVars.REDIS_PORT = String(redisPort);
|
|
300
|
+
envVars.DB_HOST = dbHost;
|
|
301
|
+
envVars.DB_PORT = String(dbPort);
|
|
302
|
+
envVars.PORT = String(containerPort);
|
|
303
|
+
resolved = interpolateEnvVars(resolved, envVars);
|
|
304
|
+
return updatePortForDocker(resolved, variablesPath);
|
|
305
|
+
}
|
|
306
|
+
/** Environment-specific transformations to resolved content. */
|
|
263
307
|
async function applyEnvironmentTransformations(resolved, environment, variablesPath) {
|
|
264
|
-
if (environment === 'docker')
|
|
265
|
-
|
|
266
|
-
resolved = await rewriteInfraEndpoints(resolved, 'docker');
|
|
267
|
-
const { getEnvHosts, getServiceHost, getServicePort, getLocalhostOverride } = require('../utils/env-endpoints');
|
|
268
|
-
const hosts = await getEnvHosts('docker');
|
|
269
|
-
const localhostOverride = getLocalhostOverride('docker');
|
|
270
|
-
const redisHost = getServiceHost(hosts.REDIS_HOST, 'docker', 'redis', localhostOverride);
|
|
271
|
-
const redisPort = await getServicePort('REDIS_PORT', 'redis', hosts, 'docker', null);
|
|
272
|
-
const dbHost = getServiceHost(hosts.DB_HOST, 'docker', 'postgres', localhostOverride);
|
|
273
|
-
const dbPort = await getServicePort('DB_PORT', 'postgres', hosts, 'docker', null);
|
|
274
|
-
const envVars = await buildEnvVarMap('docker');
|
|
275
|
-
envVars.REDIS_HOST = redisHost;
|
|
276
|
-
envVars.REDIS_PORT = String(redisPort);
|
|
277
|
-
envVars.DB_HOST = dbHost;
|
|
278
|
-
envVars.DB_PORT = String(dbPort);
|
|
279
|
-
resolved = interpolateEnvVars(resolved, envVars);
|
|
280
|
-
resolved = await updatePortForDocker(resolved, variablesPath);
|
|
281
|
-
} else if (environment === 'local') {
|
|
282
|
-
// adjustLocalEnvPortsInContent handles both PORT and infra endpoints
|
|
283
|
-
resolved = await adjustLocalEnvPortsInContent(resolved, variablesPath);
|
|
284
|
-
}
|
|
308
|
+
if (environment === 'docker') return applyDockerTransformations(resolved, variablesPath);
|
|
309
|
+
if (environment === 'local') return adjustLocalEnvPortsInContent(resolved, variablesPath);
|
|
285
310
|
return resolved;
|
|
286
311
|
}
|
|
287
312
|
|
|
288
|
-
/**
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
313
|
+
/**
|
|
314
|
+
* Generate .env content from template and secrets (no disk write).
|
|
315
|
+
* When options.envOnly is true, variablesPath is null (no application config).
|
|
316
|
+
*
|
|
317
|
+
* @param {string} appName - Application name
|
|
318
|
+
* @param {string} [secretsPath] - Path to secrets file (optional)
|
|
319
|
+
* @param {string} [environment='local'] - Environment context
|
|
320
|
+
* @param {boolean} [force=false] - Generate missing secret keys
|
|
321
|
+
* @param {Object} [options] - Optional: appPath, envOnly (env-only mode uses only env.template)
|
|
322
|
+
* @returns {Promise<string>} Resolved env content
|
|
323
|
+
*/
|
|
324
|
+
async function generateEnvContent(appName, secretsPath, environment = 'local', force = false, options = {}) {
|
|
325
|
+
const appPath = (options && options.appPath) || pathsUtil.getBuilderPath(appName);
|
|
326
|
+
const templatePath = path.join(appPath, 'env.template');
|
|
327
|
+
const variablesPath = (options && options.envOnly) ? null : resolveApplicationConfigPath(appPath);
|
|
293
328
|
const template = loadEnvTemplate(templatePath);
|
|
294
329
|
const secretsPaths = await getActualSecretsPath(secretsPath, appName);
|
|
295
330
|
if (force) {
|
|
296
|
-
const
|
|
297
|
-
await
|
|
331
|
+
const preferredPath = secretsPath ? resolveSecretsPath(secretsPath) : secretsPaths.userPath;
|
|
332
|
+
await secretsEnsure.ensureSecretsFromEnvTemplate(templatePath, { preferredFilePath: preferredPath });
|
|
298
333
|
}
|
|
299
|
-
|
|
300
334
|
const secrets = await loadSecrets(secretsPath, appName);
|
|
301
335
|
let resolved = await resolveKvReferences(template, secrets, environment, secretsPaths, appName);
|
|
302
336
|
resolved = await applyEnvironmentTransformations(resolved, environment, variablesPath);
|
|
@@ -371,73 +405,104 @@ function mergeEnvContentPreservingExisting(newContent, existingMap) {
|
|
|
371
405
|
}
|
|
372
406
|
|
|
373
407
|
/**
|
|
374
|
-
*
|
|
375
|
-
*
|
|
376
|
-
*
|
|
408
|
+
* Merges a key-value map into existing .env file content, preserving comments and blank lines.
|
|
409
|
+
* For each KEY=value line in existing content, replaces value with newMap[KEY] when the key exists
|
|
410
|
+
* in newMap. Appends any keys from newMap that did not appear in the file.
|
|
377
411
|
*
|
|
412
|
+
* @function mergeEnvMapIntoContent
|
|
413
|
+
* @param {string} existingContent - Full existing .env file content
|
|
414
|
+
* @param {Object.<string, string>} newMap - New key-to-value map (e.g. from resolved or run env)
|
|
415
|
+
* @returns {string} Merged content with comments preserved
|
|
416
|
+
*/
|
|
417
|
+
function mergeEnvMapIntoContent(existingContent, newMap) {
|
|
418
|
+
if (!newMap || Object.keys(newMap).length === 0) {
|
|
419
|
+
return typeof existingContent === 'string' ? existingContent : '';
|
|
420
|
+
}
|
|
421
|
+
const lines = (existingContent || '').split(/\r?\n/);
|
|
422
|
+
const seen = new Set();
|
|
423
|
+
const out = [];
|
|
424
|
+
for (const line of lines) {
|
|
425
|
+
const trimmed = line.trim();
|
|
426
|
+
if (!trimmed || trimmed.startsWith('#')) {
|
|
427
|
+
out.push(line);
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
const eq = trimmed.indexOf('=');
|
|
431
|
+
if (eq > 0) {
|
|
432
|
+
const key = trimmed.substring(0, eq).trim();
|
|
433
|
+
seen.add(key);
|
|
434
|
+
out.push(Object.prototype.hasOwnProperty.call(newMap, key) ? `${key}=${newMap[key]}` : line);
|
|
435
|
+
continue;
|
|
436
|
+
}
|
|
437
|
+
out.push(line);
|
|
438
|
+
}
|
|
439
|
+
for (const key of Object.keys(newMap)) {
|
|
440
|
+
if (!seen.has(key)) out.push(`${key}=${newMap[key]}`);
|
|
441
|
+
}
|
|
442
|
+
return out.join('\n');
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/**
|
|
446
|
+
* Resolves content to write for .env: merges with existing file when present.
|
|
447
|
+
* @param {string} resolved - Newly generated content
|
|
448
|
+
* @param {string} pathToPreserve - Path to existing .env to merge from (or null)
|
|
449
|
+
* @returns {string} Content to write
|
|
450
|
+
*/
|
|
451
|
+
function resolveEnvContentToWrite(resolved, pathToPreserve) {
|
|
452
|
+
if (!pathToPreserve || !fs.existsSync(pathToPreserve)) return resolved;
|
|
453
|
+
const existingContent = fs.readFileSync(pathToPreserve, 'utf8');
|
|
454
|
+
const existingMap = parseEnvContentToMap(existingContent);
|
|
455
|
+
return mergeEnvContentPreservingExisting(resolved, existingMap);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
/**
|
|
459
|
+
* Generates and writes .env file. Newly resolved values win over existing .env; extra vars in existing .env are kept.
|
|
460
|
+
* When options.envOnly is true, only env.template is used; .env is written to options.appPath.
|
|
378
461
|
* @async
|
|
379
462
|
* @function generateEnvFile
|
|
380
463
|
* @param {string} appName - Name of the application
|
|
381
464
|
* @param {string} [secretsPath] - Path to secrets file (optional)
|
|
382
465
|
* @param {string} [environment='local'] - Environment context ('local' or 'docker')
|
|
383
466
|
* @param {boolean} [force=false] - Generate missing secret keys in secrets file
|
|
384
|
-
* @param {
|
|
385
|
-
* @param {string} [preserveFromPath=null] - Path to existing .env to preserve values from (defaults to builder .env)
|
|
467
|
+
* @param {Object} [options] - Optional: appPath, envOnly, skipOutputPath, preserveFromPath
|
|
386
468
|
* @returns {Promise<string>} Path to generated .env file
|
|
387
|
-
* @throws {Error} If generation fails
|
|
388
|
-
*
|
|
389
|
-
* @example
|
|
390
|
-
* const envPath = await generateEnvFile('myapp', undefined, 'docker');
|
|
391
|
-
* // When builder/myapp/.env already exists, existing values are preserved
|
|
392
469
|
*/
|
|
393
|
-
async function generateEnvFile(appName, secretsPath, environment = 'local', force = false,
|
|
394
|
-
const
|
|
395
|
-
const
|
|
396
|
-
const
|
|
397
|
-
|
|
398
|
-
const
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
toWrite = mergeEnvContentPreservingExisting(resolved, existingMap);
|
|
470
|
+
async function generateEnvFile(appName, secretsPath, environment = 'local', force = false, options = {}) {
|
|
471
|
+
const opts = options && typeof options === 'object' ? options : {};
|
|
472
|
+
const appPath = opts.appPath || pathsUtil.getBuilderPath(appName);
|
|
473
|
+
const envOnly = !!opts.envOnly;
|
|
474
|
+
const variablesPath = envOnly ? null : resolveApplicationConfigPath(appPath);
|
|
475
|
+
const envPath = path.join(appPath, '.env');
|
|
476
|
+
|
|
477
|
+
if (envOnly) {
|
|
478
|
+
const templatePath = path.join(appPath, 'env.template');
|
|
479
|
+
if (!fs.existsSync(templatePath)) {
|
|
480
|
+
throw new Error(`env.template not found at ${templatePath}. Resolve requires env.template in the app directory.`);
|
|
481
|
+
}
|
|
406
482
|
}
|
|
407
483
|
|
|
484
|
+
const resolved = await generateEnvContent(appName, secretsPath, environment, force, { appPath, envOnly });
|
|
485
|
+
const preservePath = opts.preserveFromPath !== undefined && opts.preserveFromPath !== null ? opts.preserveFromPath : null;
|
|
486
|
+
const pathToPreserve = preservePath !== null ? preservePath : envPath;
|
|
487
|
+
const toWrite = resolveEnvContentToWrite(resolved, pathToPreserve);
|
|
408
488
|
fs.writeFileSync(envPath, toWrite, { mode: 0o600 });
|
|
409
489
|
|
|
410
|
-
|
|
411
|
-
|
|
490
|
+
if (!opts.skipOutputPath) {
|
|
491
|
+
const { processEnvVariables } = require('../utils/env-copy');
|
|
412
492
|
await processEnvVariables(envPath, variablesPath, appName, secretsPath);
|
|
413
493
|
}
|
|
414
494
|
|
|
415
495
|
return envPath;
|
|
416
496
|
}
|
|
417
497
|
|
|
418
|
-
/**
|
|
419
|
-
* Generates admin secrets for infrastructure
|
|
420
|
-
* Creates ~/.aifabrix/admin-secrets.env with Postgres and Redis credentials
|
|
421
|
-
*
|
|
422
|
-
* @async
|
|
423
|
-
* @function generateAdminSecretsEnv
|
|
424
|
-
* @param {string} [secretsPath] - Path to secrets file (optional)
|
|
425
|
-
* @returns {Promise<string>} Path to generated admin-secrets.env file
|
|
426
|
-
* @throws {Error} If generation fails
|
|
427
|
-
*
|
|
428
|
-
* @example
|
|
429
|
-
* const adminEnvPath = await generateAdminSecretsEnv('../../secrets.local.yaml');
|
|
430
|
-
* // Returns: '~/.aifabrix/admin-secrets.env'
|
|
431
|
-
*/
|
|
498
|
+
/** Generates admin secrets for infrastructure (~/.aifabrix/admin-secrets.env). Uses admin123 when no postgres password. */
|
|
432
499
|
async function generateAdminSecretsEnv(secretsPath) {
|
|
433
500
|
let secrets;
|
|
434
501
|
|
|
435
502
|
try {
|
|
436
503
|
secrets = await loadSecrets(secretsPath);
|
|
437
504
|
} catch (error) {
|
|
438
|
-
// If secrets file doesn't exist, create default secrets
|
|
439
505
|
const defaultSecretsPath = secretsPath || path.join(pathsUtil.getAifabrixHome(), 'secrets.yaml');
|
|
440
|
-
|
|
441
506
|
if (!fs.existsSync(defaultSecretsPath)) {
|
|
442
507
|
logger.log('Creating default secrets file...');
|
|
443
508
|
await createDefaultSecrets(defaultSecretsPath);
|
|
@@ -446,15 +511,14 @@ async function generateAdminSecretsEnv(secretsPath) {
|
|
|
446
511
|
throw error;
|
|
447
512
|
}
|
|
448
513
|
}
|
|
449
|
-
|
|
450
514
|
const aifabrixDir = pathsUtil.getAifabrixHome();
|
|
451
515
|
const adminEnvPath = path.join(aifabrixDir, 'admin-secrets.env');
|
|
452
|
-
|
|
453
516
|
if (!fs.existsSync(aifabrixDir)) {
|
|
454
517
|
fs.mkdirSync(aifabrixDir, { recursive: true, mode: 0o700 });
|
|
455
518
|
}
|
|
456
519
|
|
|
457
|
-
const
|
|
520
|
+
const raw = secrets['postgres-passwordKeyVault'];
|
|
521
|
+
const postgresPassword = (raw && String(raw).trim()) || 'admin123';
|
|
458
522
|
|
|
459
523
|
const adminSecrets = `# Infrastructure Admin Credentials
|
|
460
524
|
POSTGRES_PASSWORD=${postgresPassword}
|
|
@@ -477,5 +541,7 @@ module.exports = {
|
|
|
477
541
|
generateAdminSecretsEnv,
|
|
478
542
|
validateSecrets,
|
|
479
543
|
createDefaultSecrets,
|
|
480
|
-
getCanonicalSecretName
|
|
544
|
+
getCanonicalSecretName,
|
|
545
|
+
parseEnvContentToMap,
|
|
546
|
+
mergeEnvMapIntoContent
|
|
481
547
|
};
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Field Reference Validator for External Datasource
|
|
3
|
+
*
|
|
4
|
+
* Validates that field names used in indexing (embedding, uniqueKey),
|
|
5
|
+
* validation.repeatingValues[].field, and quality.rejectIf[].field exist in
|
|
6
|
+
* fieldMappings.attributes. Aligns with dataplane invalid_reference semantics.
|
|
7
|
+
*
|
|
8
|
+
* @fileoverview Offline field reference validation for AI Fabrix Builder
|
|
9
|
+
* @author AI Fabrix Team
|
|
10
|
+
* @version 2.0.0
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Validates that all field references in indexing, validation, and quality
|
|
15
|
+
* exist in fieldMappings.attributes. When fieldMappings.attributes is missing
|
|
16
|
+
* or empty, returns no errors (skip check, matching dataplane behavior).
|
|
17
|
+
*
|
|
18
|
+
* @function validateFieldReferences
|
|
19
|
+
* @param {Object} parsed - Parsed datasource object (after JSON parse)
|
|
20
|
+
* @returns {string[]} Array of error messages; empty if no invalid references
|
|
21
|
+
*
|
|
22
|
+
* @example
|
|
23
|
+
* const errors = validateFieldReferences(parsed);
|
|
24
|
+
* if (errors.length > 0) {
|
|
25
|
+
* errors.forEach(e => console.error(e));
|
|
26
|
+
* }
|
|
27
|
+
*/
|
|
28
|
+
function validateFieldReferences(parsed) {
|
|
29
|
+
const errors = [];
|
|
30
|
+
const normalizedAttributes = Object.keys(
|
|
31
|
+
parsed?.fieldMappings?.attributes ?? {}
|
|
32
|
+
);
|
|
33
|
+
|
|
34
|
+
if (normalizedAttributes.length === 0) {
|
|
35
|
+
return [];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// indexing.embedding: array of field names
|
|
39
|
+
const embedding = parsed?.indexing?.embedding;
|
|
40
|
+
if (Array.isArray(embedding)) {
|
|
41
|
+
embedding.forEach((field, i) => {
|
|
42
|
+
if (typeof field === 'string' && !normalizedAttributes.includes(field)) {
|
|
43
|
+
errors.push(
|
|
44
|
+
`indexing.embedding[${i}]: field '${field}' does not exist in fieldMappings.attributes`
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// indexing.uniqueKey: single field name
|
|
51
|
+
const uniqueKey = parsed?.indexing?.uniqueKey;
|
|
52
|
+
if (typeof uniqueKey === 'string' && uniqueKey !== '') {
|
|
53
|
+
if (!normalizedAttributes.includes(uniqueKey)) {
|
|
54
|
+
errors.push(
|
|
55
|
+
`indexing.uniqueKey: field '${uniqueKey}' does not exist in fieldMappings.attributes`
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// validation.repeatingValues[].field
|
|
61
|
+
const repeatingValues = parsed?.validation?.repeatingValues;
|
|
62
|
+
if (Array.isArray(repeatingValues)) {
|
|
63
|
+
repeatingValues.forEach((rule, index) => {
|
|
64
|
+
const field = rule?.field;
|
|
65
|
+
if (typeof field === 'string' && !normalizedAttributes.includes(field)) {
|
|
66
|
+
errors.push(
|
|
67
|
+
`validation.repeatingValues[${index}].field: field '${field}' does not exist in fieldMappings.attributes`
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// quality.rejectIf[].field
|
|
74
|
+
const rejectIf = parsed?.quality?.rejectIf;
|
|
75
|
+
if (Array.isArray(rejectIf)) {
|
|
76
|
+
rejectIf.forEach((rule, index) => {
|
|
77
|
+
const field = rule?.field;
|
|
78
|
+
if (typeof field === 'string' && !normalizedAttributes.includes(field)) {
|
|
79
|
+
errors.push(
|
|
80
|
+
`quality.rejectIf[${index}].field: field '${field}' does not exist in fieldMappings.attributes`
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return errors;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
module.exports = {
|
|
90
|
+
validateFieldReferences
|
|
91
|
+
};
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
const fs = require('fs');
|
|
12
12
|
const { loadExternalDataSourceSchema } = require('../utils/schema-loader');
|
|
13
13
|
const { formatValidationErrors } = require('../utils/error-formatter');
|
|
14
|
+
const { validateFieldReferences } = require('./field-reference-validator');
|
|
14
15
|
|
|
15
16
|
/**
|
|
16
17
|
* Validates a datasource file against external-datasource schema
|
|
@@ -48,11 +49,28 @@ async function validateDatasourceFile(filePath) {
|
|
|
48
49
|
}
|
|
49
50
|
|
|
50
51
|
const validate = loadExternalDataSourceSchema();
|
|
51
|
-
const
|
|
52
|
+
const schemaValid = validate(parsed);
|
|
53
|
+
|
|
54
|
+
if (!schemaValid) {
|
|
55
|
+
return {
|
|
56
|
+
valid: false,
|
|
57
|
+
errors: formatValidationErrors(validate.errors),
|
|
58
|
+
warnings: []
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const fieldRefErrors = validateFieldReferences(parsed);
|
|
63
|
+
if (fieldRefErrors.length > 0) {
|
|
64
|
+
return {
|
|
65
|
+
valid: false,
|
|
66
|
+
errors: fieldRefErrors,
|
|
67
|
+
warnings: []
|
|
68
|
+
};
|
|
69
|
+
}
|
|
52
70
|
|
|
53
71
|
return {
|
|
54
|
-
valid,
|
|
55
|
-
errors:
|
|
72
|
+
valid: true,
|
|
73
|
+
errors: [],
|
|
56
74
|
warnings: []
|
|
57
75
|
};
|
|
58
76
|
}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Environment deployment config and preset helpers.
|
|
3
|
+
* Parses/validates JSON config and builds preset-based config.
|
|
4
|
+
*
|
|
5
|
+
* @fileoverview Environment config for AI Fabrix Builder
|
|
6
|
+
* @author AI Fabrix Team
|
|
7
|
+
* @version 2.0.0
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
const Ajv = require('ajv');
|
|
13
|
+
const { formatValidationErrors } = require('../utils/error-formatter');
|
|
14
|
+
const environmentDeployRequestSchema = require('../schema/environment-deploy-request.schema.json');
|
|
15
|
+
|
|
16
|
+
/** Allowed preset values for --preset (case-insensitive); API expects lowercase s, m, l, xl */
|
|
17
|
+
const PRESET_VALUES = ['s', 'm', 'l', 'xl'];
|
|
18
|
+
const DEFAULT_PRESET = 's';
|
|
19
|
+
const DEFAULT_SERVICE_NAME = 'aifabrix';
|
|
20
|
+
const DEFAULT_LOCATION = 'swedencentral';
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Normalizes and validates preset from CLI (s, m, l, xl; case-insensitive)
|
|
24
|
+
* @param {string} [preset] - User-provided preset
|
|
25
|
+
* @returns {string} Normalized preset (lowercase)
|
|
26
|
+
* @throws {Error} If preset is not one of s, m, l, xl
|
|
27
|
+
*/
|
|
28
|
+
function normalizePreset(preset) {
|
|
29
|
+
const raw = (preset === null || preset === undefined || preset === '') ? DEFAULT_PRESET : String(preset).trim().toLowerCase();
|
|
30
|
+
if (!PRESET_VALUES.includes(raw)) {
|
|
31
|
+
throw new Error(
|
|
32
|
+
`Invalid preset "${preset}". Use one of: ${PRESET_VALUES.join(', ')}.\n` +
|
|
33
|
+
'Example: aifabrix env deploy dev --preset s'
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
return raw;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Builds environmentConfig from env key and preset (no config file)
|
|
41
|
+
* @param {string} validatedEnvKey - Validated environment key
|
|
42
|
+
* @param {string} preset - Normalized preset (s, m, l, xl)
|
|
43
|
+
* @returns {Object} { environmentConfig, dryRun: false }
|
|
44
|
+
*/
|
|
45
|
+
function buildEnvironmentConfigFromPreset(validatedEnvKey, preset) {
|
|
46
|
+
return {
|
|
47
|
+
environmentConfig: {
|
|
48
|
+
key: validatedEnvKey,
|
|
49
|
+
environment: validatedEnvKey,
|
|
50
|
+
preset,
|
|
51
|
+
serviceName: DEFAULT_SERVICE_NAME,
|
|
52
|
+
location: DEFAULT_LOCATION
|
|
53
|
+
},
|
|
54
|
+
dryRun: false
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Reads and parses config file; throws if missing, unreadable, or invalid structure. */
|
|
59
|
+
function parseEnvironmentConfigFile(resolvedPath) {
|
|
60
|
+
let raw;
|
|
61
|
+
try {
|
|
62
|
+
raw = fs.readFileSync(resolvedPath, 'utf8');
|
|
63
|
+
} catch (e) {
|
|
64
|
+
throw new Error(`Cannot read config file: ${resolvedPath}. ${e.message}`);
|
|
65
|
+
}
|
|
66
|
+
let parsed;
|
|
67
|
+
try {
|
|
68
|
+
parsed = JSON.parse(raw);
|
|
69
|
+
} catch (e) {
|
|
70
|
+
throw new Error(
|
|
71
|
+
`Invalid JSON in config file: ${resolvedPath}\n${e.message}\n` +
|
|
72
|
+
'Expected format: { "environmentConfig": { "key", "environment", "preset", "serviceName", "location" }, "dryRun": false }'
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
if (parsed === null || typeof parsed !== 'object') {
|
|
76
|
+
throw new Error(
|
|
77
|
+
`Config file must be a JSON object with "environmentConfig". File: ${resolvedPath}\n` +
|
|
78
|
+
'Example: { "environmentConfig": { "key": "dev", "environment": "dev", "preset": "s", "serviceName": "aifabrix", "location": "swedencentral" }, "dryRun": false }'
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
if (parsed.environmentConfig === undefined) {
|
|
82
|
+
throw new Error(
|
|
83
|
+
`Config file must contain "environmentConfig" (object). File: ${resolvedPath}\n` +
|
|
84
|
+
'Example: { "environmentConfig": { "key": "dev", "environment": "dev", "preset": "s", "serviceName": "aifabrix", "location": "swedencentral" } }'
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
if (typeof parsed.environmentConfig !== 'object' || parsed.environmentConfig === null) {
|
|
88
|
+
throw new Error(`"environmentConfig" must be an object. File: ${resolvedPath}`);
|
|
89
|
+
}
|
|
90
|
+
return parsed;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Validates parsed config against schema and returns deploy request.
|
|
95
|
+
* @param {Object} parsed - Parsed config object
|
|
96
|
+
* @param {string} resolvedPath - Path for error messages
|
|
97
|
+
* @returns {Object} { environmentConfig, dryRun? }
|
|
98
|
+
*/
|
|
99
|
+
function validateEnvironmentDeployParsed(parsed, resolvedPath) {
|
|
100
|
+
const ajv = new Ajv({ allErrors: true, strict: false });
|
|
101
|
+
const validate = ajv.compile(environmentDeployRequestSchema);
|
|
102
|
+
if (!validate(parsed)) {
|
|
103
|
+
const messages = formatValidationErrors(validate.errors);
|
|
104
|
+
throw new Error(
|
|
105
|
+
`Environment config validation failed (${resolvedPath}):\n • ${messages.join('\n • ')}\n` +
|
|
106
|
+
'Fix the config file and run the command again. See templates/infra/environment-dev.json for a valid example.'
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
environmentConfig: parsed.environmentConfig,
|
|
111
|
+
dryRun: Boolean(parsed.dryRun)
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Loads and validates environment deploy config from a JSON file
|
|
117
|
+
* @param {string} configPath - Absolute or relative path to config JSON
|
|
118
|
+
* @returns {Object} Valid deploy request { environmentConfig, dryRun? }
|
|
119
|
+
* @throws {Error} If file missing, invalid JSON, or validation fails
|
|
120
|
+
*/
|
|
121
|
+
function loadAndValidateEnvironmentDeployConfig(configPath) {
|
|
122
|
+
const resolvedPath = path.isAbsolute(configPath) ? configPath : path.resolve(process.cwd(), configPath);
|
|
123
|
+
if (!fs.existsSync(resolvedPath)) {
|
|
124
|
+
throw new Error(
|
|
125
|
+
`Environment config file not found: ${resolvedPath}\n` +
|
|
126
|
+
'Use --config <file> with a JSON file containing "environmentConfig" (e.g. templates/infra/environment-dev.json).'
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
const parsed = parseEnvironmentConfigFile(resolvedPath);
|
|
130
|
+
return validateEnvironmentDeployParsed(parsed, resolvedPath);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
module.exports = {
|
|
134
|
+
normalizePreset,
|
|
135
|
+
buildEnvironmentConfigFromPreset,
|
|
136
|
+
loadAndValidateEnvironmentDeployConfig
|
|
137
|
+
};
|