@aifabrix/builder 2.42.1 → 2.44.0

package/lib/core/secrets-ensure.js

@@ -17,7 +17,7 @@ const os = require('os');
 const config = require('./config');
 const pathsUtil = require('../utils/paths');
 const logger = require('../utils/logger');
-const { isRemoteSecretsUrl, getRemoteDevAuth } = require('../utils/remote-dev-auth');
+const remoteDevAuth = require('../utils/remote-dev-auth');
 const devApi = require('../api/dev.api');
 const {
   findMissingSecretKeys,
@@ -28,6 +28,21 @@ const {
 } = require('../utils/secrets-generator');
 const { encryptSecret } = require('../utils/secrets-encryption');
 const { loadEnvTemplate } = require('../utils/secrets-helpers');
+const {
+  buildInfraPlaceholderContext,
+  isSecretKeyAllowedEmpty,
+  getInfraSecretKeysForUpInfra
+} = require('./secrets-ensure-infra');
+const { syncLiteralKvSecretsFromCliOverrides } = require('./secrets-infra-placeholder-sync');
+
+/**
+ * Lazy require so tests that `jest.mock('../parameters/infra-parameter-catalog')` after other
+ * suites loaded secrets-ensure still see the mocked exports (avoids stale destructured refs).
+ * @returns {typeof import('../parameters/infra-parameter-catalog')}
+ */
+function infraParameterCatalogModule() {
+  return require('../parameters/infra-parameter-catalog');
+}
 
 /**
  * Expand leading ~ to home directory.
@@ -49,27 +64,30 @@ function expandTilde(filePath) {
  * - http(s) URL → remote (fallback: user file)
  * - No config → user file
  *
- * @returns {Promise<{ type: 'file'|'remote', filePath?: string, serverUrl?: string, clientCertPem?: string }>}
+ * @returns {Promise<{ type: 'file'|'remote', filePath?: string, serverUrl?: string|null, secretsEndpointUrl?: string, clientCertPem?: string|null, serverCaPem?: string|null }>}
  */
 async function resolveWriteTarget() {
   const secretsPath = await config.getSecretsPath();
-  const userFilePath = path.join(pathsUtil.getAifabrixHome(), 'secrets.local.yaml');
+  const userFilePath = pathsUtil.getPrimaryUserSecretsLocalPath();
 
   if (!secretsPath) {
     return { type: 'file', filePath: userFilePath };
   }
-  if (isRemoteSecretsUrl(secretsPath)) {
-    const auth = await getRemoteDevAuth();
+  const resolvedSecrets = await remoteDevAuth.resolveSharedSecretsEndpoint(secretsPath);
+  if (remoteDevAuth.isRemoteSecretsUrl(resolvedSecrets)) {
+    const auth = await remoteDevAuth.getRemoteDevAuth();
     return {
       type: 'remote',
       filePath: userFilePath,
-      serverUrl: secretsPath.replace(/\/+$/, ''),
-      clientCertPem: auth ? auth.clientCertPem : null
+      serverUrl: auth ? auth.serverUrl : null,
+      secretsEndpointUrl: resolvedSecrets.trim().replace(/\/+$/, ''),
+      clientCertPem: auth ? auth.clientCertPem : null,
+      serverCaPem: auth ? auth.serverCaPem : null
     };
   }
-  const filePath = path.isAbsolute(secretsPath)
-    ? secretsPath
-    : path.resolve(process.cwd(), expandTilde(secretsPath));
+  const filePath = path.isAbsolute(resolvedSecrets)
+    ? resolvedSecrets
+    : path.resolve(process.cwd(), expandTilde(resolvedSecrets));
   return { type: 'file', filePath };
 }
 
@@ -85,7 +103,12 @@ async function loadExistingFromTarget(target) {
   }
   if (target.type === 'remote' && target.serverUrl && target.clientCertPem) {
     try {
-      const items = await devApi.listSecrets(target.serverUrl, target.clientCertPem);
+      const items = await devApi.listSecrets(
+        target.serverUrl,
+        target.clientCertPem,
+        target.serverCaPem || undefined,
+        target.secretsEndpointUrl
+      );
       if (!Array.isArray(items)) return {};
       const obj = {};
       for (const item of items) {
@@ -134,11 +157,12 @@ async function writeSecretToFile(filePath, key, value, encryptionKey) {
  * @param {string} key - Secret key
  * @param {Object} suggested - Map of suggested values
  * @param {boolean} emptyForCredentials - Use empty string if true
+ * @param {Record<string, string>|undefined} placeholderContext - Catalog defaults + CLI merged map
  * @returns {string}
  */
-function valueForKey(key, suggested, emptyForCredentials) {
+function valueForKey(key, suggested, emptyForCredentials, placeholderContext) {
   if (key in suggested) return String(suggested[key]);
-  return emptyForCredentials ? '' : generateSecretValue(key);
+  return emptyForCredentials ? '' : generateSecretValue(key, placeholderContext);
 }
 
 /**
@@ -150,12 +174,18 @@ function valueForKey(key, suggested, emptyForCredentials) {
  * @param {string|null} encryptionKey - Encryption key for file fallback or null
  * @returns {Promise<string[]>}
  */
-async function addSecretsRemote(target, toAdd, suggested, added, encryptionKey) {
+async function addSecretsRemote(target, toAdd, suggested, added, encryptionKey, placeholderContext) {
   const emptyForCredentials = false;
   for (const key of toAdd) {
-    const value = valueForKey(key, suggested, emptyForCredentials);
+    const value = valueForKey(key, suggested, emptyForCredentials, placeholderContext);
     try {
-      await devApi.addSecret(target.serverUrl, target.clientCertPem, { key, value });
+      await devApi.addSecret(
+        target.serverUrl,
+        target.clientCertPem,
+        { key, value },
+        target.serverCaPem || undefined,
+        target.secretsEndpointUrl
+      );
       added.push(key);
     } catch (err) {
       logger.warn(`Remote secret "${key}" failed (${err.message}); writing to local file.`);
@@ -168,22 +198,33 @@ async function addSecretsRemote(target, toAdd, suggested, added, encryptionKey)
 
 /**
  * Add secrets to file (with optional encryption).
- * @param {string} filePath - File path
- * @param {string[]} toAdd - Keys to add
- * @param {Object} suggested - Suggested values
- * @param {boolean} emptyForCredentials - Use empty for new values
- * @param {string|null} encryptionKey - Encryption key or null
- * @param {string[]} added - Array to push added keys to
+ * @param {Object} batch
+ * @param {string} batch.filePath
+ * @param {string[]} batch.toAdd
+ * @param {Object} batch.suggested
+ * @param {boolean} batch.emptyForCredentials
+ * @param {string|null} batch.encryptionKey
+ * @param {string[]} batch.added
+ * @param {Record<string, string>|undefined} batch.placeholderContext
  * @returns {Promise<string[]>}
  */
-async function addSecretsToFile(filePath, toAdd, suggested, emptyForCredentials, encryptionKey, added) {
+async function addSecretsToFile(batch) {
+  const {
+    filePath,
+    toAdd,
+    suggested,
+    emptyForCredentials,
+    encryptionKey,
+    added,
+    placeholderContext
+  } = batch;
   for (const key of toAdd) {
-    const value = valueForKey(key, suggested, emptyForCredentials);
+    const value = valueForKey(key, suggested, emptyForCredentials, placeholderContext);
     await writeSecretToFile(filePath, key, value, encryptionKey);
     added.push(key);
   }
   if (added.length > 0) {
-    logger.log(`✓ Ensured ${added.length} secret key(s): ${added.join(', ')}`);
+    logger.log(`✔ Ensured ${added.length} secret key(s): ${added.join(', ')}`);
   }
   return added;
 }
@@ -210,13 +251,14 @@ async function ensureSecretsForKeys(keys, options = {}) {
   const suggested = options.suggestedValues && typeof options.suggestedValues === 'object'
     ? options.suggestedValues
     : {};
+  const placeholderContext = options.placeholderContext;
 
   const target = options._targetOverride || await resolveWriteTarget();
   const existing = await loadExistingFromTarget(target);
   const toAdd = keys.filter((k) => {
     const v = existing[k];
     const missingOrEmpty = v === undefined || v === null || (typeof v === 'string' && v.trim() === '');
-    return missingOrEmpty && !KEYS_ALLOWED_EMPTY.has(k);
+    return missingOrEmpty && !isSecretKeyAllowedEmpty(k);
   });
   if (toAdd.length === 0) return [];
 
@@ -224,9 +266,17 @@ async function ensureSecretsForKeys(keys, options = {}) {
   const added = [];
 
   if (target.type === 'remote' && target.serverUrl && target.clientCertPem) {
-    return addSecretsRemote(target, toAdd, suggested, added, encryptionKey);
+    return addSecretsRemote(target, toAdd, suggested, added, encryptionKey, placeholderContext);
   }
-  return addSecretsToFile(target.filePath, toAdd, suggested, emptyForCredentials, encryptionKey, added);
+  return addSecretsToFile({
+    filePath: target.filePath,
+    toAdd,
+    suggested,
+    emptyForCredentials,
+    encryptionKey,
+    added,
+    placeholderContext
+  });
 }
 
 /**
@@ -270,33 +320,6 @@ async function ensureSecretsFromEnvTemplate(envTemplatePathOrContent, options =
   return ensureSecretsForKeys(missingKeys, { ...options, _targetOverride: target });
 }
 
-/**
- * Infra secret keys used by createDefaultSecrets / keyvault.md for up-infra.
- * Includes miso-controller DB keys so ensureMisoInitScript can read from store.
- *
- * postgres-passwordKeyVault: Postgres superuser/admin password for local Docker Postgres,
- * PgAdmin, and Redis Commander (see generateAdminSecretsEnv in lib/core/secrets.js).
- *
- * @type {string[]}
- */
-const INFRA_SECRET_KEYS = [
-  'postgres-passwordKeyVault',
-  'redis-passwordKeyVault',
-  'redis-url',
-  'keycloak-admin-passwordKeyVault',
-  'keycloak-server-url',
-  'databases-miso-controller-0-passwordKeyVault',
-  'databases-miso-controller-0-urlKeyVault'
-];
-
-/**
- * Keys that are valid when empty (e.g. no Redis password in local dev).
- * We do not backfill these when empty, so the file is not appended with a duplicate key.
- *
- * @type {Set<string>}
- */
-const KEYS_ALLOWED_EMPTY = new Set(['redis-passwordKeyVault']);
-
 /**
  * Write one key-value to a file store (load, merge, optionally encrypt, save).
  * @param {string} filePath - Secrets file path
@@ -325,7 +348,7 @@ async function writeSecretToStoreFile(filePath, key, strValue) {
 
 /**
  * Write a single secret to the configured store (overwrites if key exists).
- * Used when syncing e.g. postgres-passwordKeyVault after --adminPwd override.
+ * Used when syncing e.g. postgres-passwordKeyVault after --adminPassword override.
  *
  * @async
  * @function setSecretInStore
@@ -339,7 +362,13 @@ async function setSecretInStore(key, value) {
   const strValue = typeof value === 'string' ? value : String(value);
   if (target.type === 'remote' && target.serverUrl && target.clientCertPem) {
     try {
-      await devApi.addSecret(target.serverUrl, target.clientCertPem, { key, value: strValue });
+      await devApi.addSecret(
+        target.serverUrl,
+        target.clientCertPem,
+        { key, value: strValue },
+        target.serverCaPem || undefined,
+        target.secretsEndpointUrl
+      );
     } catch (err) {
       logger.warn(`Could not sync secret "${key}" to remote store: ${err.message}`);
       const encryptionKey = await config.getSecretsEncryptionKey();
@@ -356,15 +385,27 @@ async function setSecretInStore(key, value) {
  * @async
  * @function ensureInfraSecrets
  * @param {Object} [options] - Options
- * @param {string} [options.adminPwd] - Override for postgres-passwordKeyVault when creating new secrets
+ * @param {string} [options.adminPassword] - Overrides {{adminPassword}} (and alias adminPwd)
+ * @param {string} [options.adminPwd] - Alias for adminPassword
+ * @param {string} [options.adminEmail] - Overrides {{adminEmail}}
+ * @param {string} [options.userPassword] - Overrides {{userPassword}}
+ * @param {boolean} [options.tlsEnabled] - Merged into catalog placeholders: TLS_ENABLED / HTTP_ENABLED (HTTP is opposite of TLS; default TLS off)
  * @returns {Promise<string[]>} Keys that were added
+ *
+ * After backfill, non-empty `--adminPassword` / `--userPassword` / `--adminEmail` update all catalog
+ * literals that reference the same `{{placeholder}}` (see `secrets-infra-placeholder-sync.js`).
  */
 async function ensureInfraSecrets(options = {}) {
-  const suggested = {};
-  if (options.adminPwd && typeof options.adminPwd === 'string' && options.adminPwd.trim() !== '') {
-    suggested['postgres-passwordKeyVault'] = options.adminPwd.trim();
-  }
-  return ensureSecretsForKeys(INFRA_SECRET_KEYS, { suggestedValues: suggested });
+  const keys = getInfraSecretKeysForUpInfra();
+  const placeholderContext = buildInfraPlaceholderContext(options);
+  const added = await ensureSecretsForKeys(keys, { placeholderContext });
+  await syncLiteralKvSecretsFromCliOverrides(
+    options,
+    placeholderContext,
+    setSecretInStore,
+    infraParameterCatalogModule
+  );
+  return added;
 }
 
 module.exports = {
@@ -374,5 +415,20 @@ module.exports = {
   setSecretInStore,
   resolveWriteTarget,
   loadExistingFromTarget,
-  INFRA_SECRET_KEYS
+  getInfraSecretKeysForUpInfra,
+  isSecretKeyAllowedEmpty,
+  buildInfraPlaceholderContext
 };
+
+/**
+ * @deprecated Prefer getInfraSecretKeysForUpInfra(). Names only: relaxed read of infra.parameter.yaml
+ * (standardUpInfraEnsureKeys + exact keys with ensureOn upInfra), not workspace discovery.
+ */
+Object.defineProperty(module.exports, 'INFRA_SECRET_KEYS', {
+  enumerable: true,
+  get() {
+    const cat = infraParameterCatalogModule();
+    const bundledYaml = path.join(__dirname, '..', 'schema', 'infra.parameter.yaml');
+    return cat.readRelaxedUpInfraEnsureKeyList(bundledYaml) || [];
+  }
+});

package/lib/core/secrets-env-write.js

@@ -47,7 +47,33 @@ function getSecretForEnvVar(secrets, preferred, alternate) {
 }
 
 /**
- * Inject NPM_TOKEN and PYPI_TOKEN from loaded secrets into content when missing.
+ * Append KEY=value as its own line (avoids joining with previous line when content ended with newline).
+ * @param {string} content
+ * @param {string} key
+ * @param {string} value
+ * @returns {string}
+ */
+function appendEnvLine(content, key, value) {
+  const line = `${key}=${value}`;
+  if (!content || content.length === 0) return `${line}\n`;
+  return `${content.replace(/\s+$/, '')}\n${line}\n`;
+}
+
+/**
+ * Non-empty process.env for registry tokens when template has no kv://BASH_* line (inject fallback).
+ * @param {string} name
+ * @returns {string|null}
+ */
+function getProcessEnvToken(name) {
+  const raw = process.env[name];
+  if (raw === undefined || raw === null) return null;
+  const t = String(raw).trim();
+  return t.length > 0 ? t : null;
+}
+
+/**
+ * Inject NPM_TOKEN and PYPI_TOKEN from loaded secrets, then from process.env when still missing.
+ * kv://BASH_* refs are resolved in secrets-helpers (process.env suffix after BASH_) before this runs.
  * @param {string} content - .env-style content
  * @param {string|null} secretsPath - Path to secrets file
  * @param {string} appName - Application name
@@ -59,12 +85,14 @@ async function injectRegistryTokens(content, secretsPath, appName) {
     const loadedSecrets = await secrets.loadSecrets(secretsPath, appName);
     let out = content || '';
     if (!envContentHasKey(out, 'NPM_TOKEN')) {
-      const v = getSecretForEnvVar(loadedSecrets, 'NPM_TOKEN', 'npm_token');
-      if (v) out = out.trimEnd() + (out.endsWith('\n') ? '' : '\n') + `NPM_TOKEN=${v}\n`;
+      let v = getSecretForEnvVar(loadedSecrets, 'NPM_TOKEN', 'npm_token');
+      if (!v) v = getProcessEnvToken('NPM_TOKEN');
+      if (v) out = appendEnvLine(out, 'NPM_TOKEN', v);
     }
     if (!envContentHasKey(out, 'PYPI_TOKEN')) {
-      const v = getSecretForEnvVar(loadedSecrets, 'PYPI_TOKEN', 'pypi_token');
-      if (v) out = out.trimEnd() + (out.endsWith('\n') ? '' : '\n') + `PYPI_TOKEN=${v}\n`;
+      let v = getSecretForEnvVar(loadedSecrets, 'PYPI_TOKEN', 'pypi_token');
+      if (!v) v = getProcessEnvToken('PYPI_TOKEN');
+      if (v) out = appendEnvLine(out, 'PYPI_TOKEN', v);
     }
     return out;
   } catch {
@@ -95,7 +123,7 @@ function parseEnvContentToMap(content) {
 
 /**
  * Resolve .env in memory and write only to envOutputPath or temp (no builder/ or integration/).
- * Injects NPM_TOKEN and PYPI_TOKEN from secrets when missing so shell/install/test have registry tokens.
+ * Injects NPM_TOKEN and PYPI_TOKEN from secrets when missing, then from process.env, so shell/install/test/build can use exported tokens.
  *
  * @async
  * @function resolveAndWriteEnvFile
@@ -136,7 +164,7 @@ async function resolveAndWriteEnvFile(appName, options = {}) {
 /**
  * Resolve app env (template + kv:// secrets) and return as key-value map.
  * Used by build to pass NPM_TOKEN/PYPI_TOKEN as Docker build-args.
- * Injects NPM_TOKEN/PYPI_TOKEN from secrets when missing (same as resolveAndWriteEnvFile).
+ * Injects from secrets when missing, then from process.env if set (e.g. exported NPM_TOKEN in shell).
  *
  * @async
  * @function resolveAndGetEnvMap

package/lib/core/secrets-infra-placeholder-sync.js

@@ -0,0 +1,61 @@
+/**
+ * @fileoverview Sync secrets store with up-infra CLI placeholder overrides (catalog-driven).
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const logger = require('../utils/logger');
+
+/**
+ * When CLI passes --adminPassword, --userPassword, or --adminEmail, overwrite every catalog literal
+ * that embeds the matching {{placeholder}}. ensureSecretsForKeys only backfills missing keys; this
+ * keeps the store aligned with explicit CLI overrides (keys come from infra.parameter.yaml).
+ *
+ * @param {Object} options - Same flags as ensureInfraSecrets
+ * @param {Record<string, string>} placeholderContext - Merged catalog defaults + CLI
+ * @param {(key: string, value: string) => Promise<void>} setSecretInStore
+ * @param {() => typeof import('../parameters/infra-parameter-catalog')} getCatalogModule - Lazy catalog require
+ * @returns {Promise<void>}
+ */
+async function syncLiteralKvSecretsFromCliOverrides(
+  options,
+  placeholderContext,
+  setSecretInStore,
+  getCatalogModule
+) {
+  const crypto = require('crypto');
+  const ipc = getCatalogModule();
+  let catalog;
+  try {
+    catalog = ipc.getInfraParameterCatalog();
+  } catch {
+    return;
+  }
+
+  const runOne = async(placeholder, cliValue) => {
+    const trimmed = String(cliValue || '').trim();
+    if (!trimmed) return;
+    const keys = ipc.listKvKeysWithLiteralPlaceholder(catalog, placeholder);
+    if (keys.length === 0) return;
+    for (const key of keys) {
+      const entry = catalog.findEntryForKey(key);
+      if (!entry) continue;
+      const val = ipc.generateValueFromCatalogEntry(key, entry, crypto, placeholderContext);
+      await setSecretInStore(key, val);
+    }
+    logger.log(
+      `Updated ${keys.length} secret(s) in store (catalog literals using {{${placeholder}}}).`
+    );
+  };
+
+  await runOne(
+    'adminPassword',
+    options.adminPassword || options.adminPwd || ''
+  );
+  await runOne('userPassword', options.userPassword || '');
+  await runOne('adminEmail', options.adminEmail || '');
+}
+
+module.exports = {
+  syncLiteralKvSecretsFromCliOverrides
+};