@aifabrix/builder 2.43.0 → 2.44.0

package/lib/utils/resolve-docker-image-ref.js

@@ -0,0 +1,124 @@
+/**
+ * Resolve Docker repository path and tag from application config and optional CLI overrides.
+ * Precedence: --image (full ref), --registry CLI, image.registry in manifest, else unqualified name.
+ *
+ * For refs like localhost:5000/repo without an explicit tag, prefer --image with :tag (parse ambiguity).
+ *
+ * @fileoverview Shared Docker image reference resolution for run, compose, and version checks
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+'use strict';
+
+const { parseImageOverride } = require('./parse-image-ref');
+
+/**
+ * Repository path without host (same rules as compose-generator getImageName).
+ * @param {Object} appConfig - Application configuration
+ * @param {string} appName - Application name fallback
+ * @returns {string}
+ */
+function getRepositoryPathFromConfig(appConfig, appName) {
+  if (!appConfig || typeof appConfig !== 'object') {
+    return appName;
+  }
+  if (typeof appConfig.image === 'string') {
+    return appConfig.image.split(':')[0];
+  }
+  if (appConfig.image?.name) {
+    return appConfig.image.name;
+  }
+  if (appConfig.app?.key) {
+    return appConfig.app.key;
+  }
+  return appName;
+}
+
+/**
+ * @param {Object} [appConfig]
+ * @returns {string}
+ */
+function imageTagFromConfig(appConfig) {
+  return (appConfig && appConfig.image && appConfig.image.tag) || 'latest';
+}
+
+/**
+ * Trim and strip trailing slashes from a registry host/prefix. Empty/whitespace → ''.
+ * @param {string|undefined|null|number} registry - Registry host or prefix
+ * @returns {string}
+ */
+function normalizeDockerRegistryPrefix(registry) {
+  if (registry === null || registry === undefined) {
+    return '';
+  }
+  if (typeof registry !== 'string') {
+    return normalizeDockerRegistryPrefix(String(registry));
+  }
+  const t = registry.trim();
+  if (!t) {
+    return '';
+  }
+  return t.replace(/\/+$/, '');
+}
+
+/**
+ * Effective image repository (may include registry prefix) and tag for Docker.
+ * @param {string} appName - Application name
+ * @param {Object} appConfig - Loaded application manifest
+ * @param {Object} [runOptions] - Run/deploy options
+ * @param {string} [runOptions.image] - Full image ref override
+ * @param {string} [runOptions.registry] - CLI registry prefix (wins over manifest)
+ * @returns {{ imageName: string, imageTag: string }}
+ */
+function resolveDockerImageRef(appName, appConfig, runOptions = {}) {
+  const opts = runOptions || {};
+  if (opts.image) {
+    const parsed = parseImageOverride(opts.image);
+    return {
+      imageName: parsed ? parsed.name : getRepositoryPathFromConfig(appConfig, appName),
+      imageTag: parsed ? parsed.tag : imageTagFromConfig(appConfig)
+    };
+  }
+
+  const baseRepo = getRepositoryPathFromConfig(appConfig, appName);
+  const imageTag = imageTagFromConfig(appConfig);
+  const prefix =
+    normalizeDockerRegistryPrefix(opts.registry) ||
+    normalizeDockerRegistryPrefix(appConfig?.image?.registry ?? '');
+  if (prefix) {
+    return { imageName: `${prefix}/${baseRepo}`, imageTag };
+  }
+  return { imageName: baseRepo, imageTag };
+}
+
+/**
+ * Full image string for compose when manifest/CLI registry applies; else null (use template defaults).
+ * @param {string} appName - Application name
+ * @param {Object} appConfig - Application configuration
+ * @param {Object} [options] - Run options (image, imageOverride, tag, registry)
+ * @returns {string|null}
+ */
+function resolveComposeImageOverrideString(appName, appConfig, options = {}) {
+  if (options.image) return options.image;
+  if (options.imageOverride) return options.imageOverride;
+  const runOpts = { registry: options.registry, image: undefined };
+  if (options.tag) {
+    const { imageName } = resolveDockerImageRef(appName, appConfig, runOpts);
+    return `${imageName}:${options.tag}`;
+  }
+  const { imageName, imageTag } = resolveDockerImageRef(appName, appConfig, runOpts);
+  const shortName = getRepositoryPathFromConfig(appConfig, appName);
+  const shortTag = imageTagFromConfig(appConfig);
+  if (imageName === shortName && imageTag === shortTag) {
+    return null;
+  }
+  return `${imageName}:${imageTag}`;
+}
+
+module.exports = {
+  resolveDockerImageRef,
+  resolveComposeImageOverrideString,
+  normalizeDockerRegistryPrefix,
+  getRepositoryPathFromConfig
+};

package/lib/utils/schema-loader.js

@@ -9,9 +9,10 @@
  * @version 2.0.0
  */
 
-const fs = require('fs');
+const fsRealSync = require('../internal/fs-real-sync');
 const path = require('path');
 const Ajv = require('ajv');
+const addFormats = require('ajv-formats');
 
 // Cache for compiled validators
 // These are reset when module is reloaded (for testing)
@@ -46,11 +47,14 @@ function loadExternalSystemSchema() {
 
   const schemaPath = path.join(__dirname, '..', 'schema', 'external-system.schema.json');
 
-  if (!fs.existsSync(schemaPath)) {
-    throw new Error(`External system schema not found: ${schemaPath}`);
+  if (!fsRealSync.existsSync(schemaPath)) {
+    throw new Error(
+      `External system schema not found: ${schemaPath}. ` +
+        'Ensure the file exists (tracked under lib/schema/); run git checkout HEAD -- lib/schema/external-system.schema.json if your tree is incomplete.'
+    );
   }
 
-  const schemaContent = fs.readFileSync(schemaPath, 'utf8');
+  const schemaContent = fsRealSync.readFileSync(schemaPath, 'utf8');
   let schema;
 
   try {
@@ -60,6 +64,7 @@ function loadExternalSystemSchema() {
   }
 
   const ajv = new Ajv({ allErrors: true, strict: false });
+  addFormats(ajv);
   externalSystemValidator = ajv.compile(schema);
 
   return externalSystemValidator;
@@ -84,11 +89,14 @@ function loadExternalDataSourceSchema() {
 
   const schemaPath = path.join(__dirname, '..', 'schema', 'external-datasource.schema.json');
 
-  if (!fs.existsSync(schemaPath)) {
-    throw new Error(`External datasource schema not found: ${schemaPath}`);
+  if (!fsRealSync.existsSync(schemaPath)) {
+    throw new Error(
+      `External datasource schema not found: ${schemaPath}. ` +
+        'Ensure the file exists (tracked under lib/schema/); run git checkout HEAD -- lib/schema/external-datasource.schema.json if your tree is incomplete.'
+    );
   }
 
-  const schemaContent = fs.readFileSync(schemaPath, 'utf8');
+  const schemaContent = fsRealSync.readFileSync(schemaPath, 'utf8');
   let schema;
 
   try {
@@ -106,6 +114,11 @@ function loadExternalDataSourceSchema() {
   }
 
   const ajv = new Ajv({ allErrors: true, strict: false, strictSchema: false });
+  addFormats(ajv);
+  // external-datasource.schema.json references these by $id (aifabrix://schema/type/*)
+  ajv.addSchema(require('../schema/type/document-storage.json'));
+  ajv.addSchema(require('../schema/type/message-service.json'));
+  ajv.addSchema(require('../schema/type/vector-store.json'));
   externalDataSourceValidator = ajv.compile(schemaToCompile);
 
   return externalDataSourceValidator;
@@ -250,10 +263,10 @@ function readAndParseFileContent(filePath, content) {
   let fileContent = content;
 
   if (!fileContent) {
-    if (!fs.existsSync(filePath)) {
+    if (!fsRealSync.existsSync(filePath)) {
       throw new Error(`File not found: ${filePath}`);
     }
-    fileContent = fs.readFileSync(filePath, 'utf8');
+    fileContent = fsRealSync.readFileSync(filePath, 'utf8');
   }
 
   try {

package/lib/utils/secrets-bash-kv.js

@@ -0,0 +1,25 @@
+/**
+ * kv://BASH_<NAME> fallback: use process.env.<NAME>, then process.env.BASH_<NAME> (shared BASH_ keys).
+ * @fileoverview
+ */
+'use strict';
+
+/**
+ * @param {string} pathStr - Flat kv path (no slashes)
+ * @returns {string|undefined}
+ */
+function resolveBashKvFromProcessEnv(pathStr) {
+  if (!pathStr || typeof pathStr !== 'string' || pathStr.includes('/')) return undefined;
+  if (!pathStr.startsWith('BASH_')) return undefined;
+  const suffix = pathStr.slice(5);
+  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(suffix)) return undefined;
+  const pick = k => {
+    const raw = process.env[k];
+    if (raw === undefined || raw === null) return undefined;
+    const t = String(raw).trim();
+    return t.length > 0 ? t : undefined;
+  };
+  return pick(suffix) ?? pick(pathStr);
+}
+
+module.exports = { resolveBashKvFromProcessEnv };

package/lib/utils/secrets-generator.js

@@ -16,6 +16,17 @@ const yaml = require('js-yaml');
 const crypto = require('crypto');
 const logger = require('./logger');
 const pathsUtil = require('./paths');
+const { mergeFlatSecretsYamlPreservingComments } = require('./secrets-yaml-preserve');
+const {
+  generateDatabasePasswordValueForKey,
+  generateDatabaseUrlValueForKey,
+  parseDatabaseSecretKey
+} = require('../parameters/database-secret-values');
+
+/** Lazy require so tests can jest.spyOn getInfraParameterCatalog after module load. */
+function infraParameterCatalogModule() {
+  return require('../parameters/infra-parameter-catalog');
+}
 
 /**
  * Parse key-value pairs from YAML-like lines (last occurrence wins per key).
@@ -44,7 +55,9 @@ function parseYamlKeyValueLines(content) {
 /**
  * Parse YAML content tolerating duplicate keys (last occurrence wins).
  * Use for secrets files that may have been appended to repeatedly.
- * Tries yaml.load first; on "duplicate key" error falls back to line-by-line parse.
+ * Tries yaml.load first; on duplicate-key errors or other parse failures (e.g. `{}` on line 1
+ * then `key: value` lines — invalid multi-document without `---`) falls back to line-by-line
+ * parse for flat key: value secrets files.
  *
  * @param {string} content - Raw YAML content
  * @returns {Object} Parsed object (last value wins for duplicate keys)
@@ -56,13 +69,11 @@ function loadYamlTolerantOfDuplicateKeys(content) {
   try {
     const parsed = yaml.load(content);
     return parsed && typeof parsed === 'object' ? parsed : {};
-  } catch (err) {
-    const msg = err.message || '';
-    if (!msg.includes('duplicate') && !msg.includes('duplicated mapping')) {
-      throw err;
-    }
+  } catch {
+    // Duplicate keys, `{}` + appended YAML (invalid multi-doc), or other bad merges —
+    // flat secrets files recover via line parse (last key wins).
+    return parseYamlKeyValueLines(content);
   }
-  return parseYamlKeyValueLines(content);
 }
 
 /**
@@ -104,18 +115,37 @@ function findMissingSecretKeys(envTemplate, existingSecrets) {
   return missingKeys;
 }
 
+/**
+ * Resolve app directory for database key generation (application.yaml lookup).
+ * @param {string} key - Secret key
+ * @returns {string|null}
+ */
+function resolveAppDirForDatabaseKey(key) {
+  const parsed = parseDatabaseSecretKey(key);
+  if (!parsed) return null;
+  try {
+    const builderPath = pathsUtil.getBuilderPath(parsed.appKey);
+    if (fs.existsSync(builderPath)) return builderPath;
+  } catch {
+    /* ignore */
+  }
+  try {
+    const intPath = pathsUtil.getIntegrationPath(parsed.appKey);
+    if (fs.existsSync(intPath)) return intPath;
+  } catch {
+    /* ignore */
+  }
+  return null;
+}
+
 /**
  * Generate database password value for a key (databases-*-passwordKeyVault)
  * @param {string} key - Secret key name
  * @returns {string|null} Password string or null if key does not match
  */
 function generateDbPasswordValue(key) {
-  const dbPasswordMatch = key.match(/^databases-([a-z0-9-_]+)-\d+-passwordKeyVault$/i);
-  if (!dbPasswordMatch) return null;
-  const appName = dbPasswordMatch[1];
-  if (appName === 'miso-controller') return 'miso_pass123';
-  const dbName = appName.replace(/-/g, '_');
-  return `${dbName}_pass123`;
+  const appDir = resolveAppDirForDatabaseKey(key);
+  return generateDatabasePasswordValueForKey(key, appDir);
 }
 
 /**
@@ -124,44 +154,98 @@ function generateDbPasswordValue(key) {
  * @returns {string|null} URL string or null if key does not match
  */
 function generateDbUrlValue(key) {
-  const dbUrlMatch = key.match(/^databases-([a-z0-9-_]+)-\d+-urlKeyVault$/i);
-  if (!dbUrlMatch) return null;
-  const appName = dbUrlMatch[1];
-  if (appName === 'miso-controller') {
-    return 'postgresql://miso_user:miso_pass123@${DB_HOST}:${DB_PORT}/miso';
+  const appDir = resolveAppDirForDatabaseKey(key);
+  return generateDatabaseUrlValueForKey(key, appDir);
+}
+
+/**
+ * @param {string} key - Secret key
+ * @param {Record<string, string>|undefined} placeholderContext - Merged infra.parameter.yaml defaults + CLI overrides
+ * @returns {string|undefined} Value if catalog handled key; undefined to use legacy rules
+ */
+function tryGenerateSecretValueFromCatalog(key, placeholderContext) {
+  let catalogEntry;
+  try {
+    catalogEntry = infraParameterCatalogModule().getInfraParameterCatalog().findEntryForKey(key);
+  } catch (err) {
+    logger.warn(`infra parameter catalog unavailable (${err.message}); using legacy generators.`);
+    return undefined;
+  }
+  if (!catalogEntry || !catalogEntry.generator) return undefined;
+  const g = catalogEntry.generator.type;
+  if (g === 'databasePassword') {
+    const v = generateDbPasswordValue(key);
+    return v !== null ? v : undefined;
+  }
+  if (g === 'databaseUrl') {
+    const v = generateDbUrlValue(key);
+    return v !== null ? v : undefined;
   }
-  const dbName = appName.replace(/-/g, '_');
-  return `postgresql://${dbName}_user:${dbName}_pass123@\${DB_HOST}:\${DB_PORT}/${dbName}`;
+  if (
+    g === 'randomBytes32' ||
+    g === 'randomAlphanumeric' ||
+    g === 'password' ||
+    g === 'emptyString' ||
+    g === 'emptyAllowed' ||
+    g === 'literal'
+  ) {
+    return infraParameterCatalogModule().generateValueFromCatalogEntry(key, catalogEntry, crypto, placeholderContext);
+  }
+  return undefined;
 }
 
 /**
- * Generates secret value based on key name
- * @function generateSecretValue
- * @param {string} key - Secret key name
- * @returns {string} Generated secret value
+ * Last-resort generation when infra.parameter.yaml cannot be loaded (e.g. tests, broken install).
+ * @param {string} key
+ * @returns {string}
  */
-function generateSecretValue(key) {
+function legacySecretValueWhenCatalogUnavailable(key) {
   const keyLower = key.toLowerCase();
-
   if (keyLower.includes('password')) {
     const dbPassword = generateDbPasswordValue(key);
     if (dbPassword !== null) return dbPassword;
     return crypto.randomBytes(32).toString('base64');
   }
-
   if (keyLower.includes('url') || keyLower.includes('uri')) {
     const dbUrl = generateDbUrlValue(key);
     if (dbUrl !== null) return dbUrl;
     return '';
   }
-
   if (keyLower.includes('key') || keyLower.includes('secret') || keyLower.includes('token')) {
     return crypto.randomBytes(32).toString('base64');
   }
-
   return '';
 }
 
+/**
+ * @param {string} key - Secret key name
+ * @param {Record<string, string>|undefined} placeholderContext - Optional merged catalog defaults + CLI (e.g. up-infra)
+ * @returns {string} Generated secret value
+ */
+function generateSecretValue(key, placeholderContext) {
+  const fromCatalog = tryGenerateSecretValueFromCatalog(key, placeholderContext);
+  if (fromCatalog !== undefined) return fromCatalog;
+
+  const dbPassword = generateDbPasswordValue(key);
+  if (dbPassword !== null) return dbPassword;
+  const dbUrl = generateDbUrlValue(key);
+  if (dbUrl !== null) return dbUrl;
+
+  try {
+    infraParameterCatalogModule().getInfraParameterCatalog();
+  } catch {
+    logger.warn(
+      `Secret key "${key}": infra parameter catalog unavailable; using legacy name heuristics.`
+    );
+    return legacySecretValueWhenCatalogUnavailable(key);
+  }
+
+  throw new Error(
+    `Cannot generate secret for key "${key}": no matching rule in infra.parameter.yaml. ` +
+      'Add a catalog entry or run `aifabrix parameters validate` after fixing env.template references.'
+  );
+}
+
 /**
  * Loads existing secrets from file
  * @function loadExistingSecrets
@@ -196,6 +280,23 @@ function saveSecretsFile(resolvedPath, secrets) {
     fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
   }
 
+  if (fs.existsSync(resolvedPath)) {
+    try {
+      const existing = fs.readFileSync(resolvedPath, 'utf8');
+      const mergedContent = mergeFlatSecretsYamlPreservingComments(existing, secrets, {
+        yaml,
+        dumpOpts: YAML_DUMP_OPTS
+      });
+      if (mergedContent !== null) {
+        fs.writeFileSync(resolvedPath, mergedContent + (mergedContent.endsWith('\n') ? '' : '\n'), { mode: 0o600 });
+        return;
+      }
+    } catch (err) {
+      logger.warn(`Could not preserve comments when writing secrets file: ${err.message}; rewriting YAML.`);
+      // fall through to dump overwrite
+    }
+  }
+
   const yamlContent = yaml.dump(secrets, {
     indent: 2,
     lineWidth: -1,
@@ -208,6 +309,20 @@ function saveSecretsFile(resolvedPath, secrets) {
 
 const YAML_DUMP_OPTS = { indent: 2, lineWidth: -1, noRefs: true, sortKeys: false };
 
+/**
+ * @param {string} resolvedPath
+ * @returns {string}
+ */
+function readSecretsFileUtf8OrWarnEmpty(resolvedPath) {
+  try {
+    const raw = fs.readFileSync(resolvedPath, 'utf8');
+    return typeof raw === 'string' ? raw : '';
+  } catch (err) {
+    logger.warn(`Could not read existing secrets file: ${err.message}; appending new keys only.`);
+    return '';
+  }
+}
+
 /**
  * Merges secret keys into the secrets file (load existing, merge, overwrite file).
  * Use when setting or updating keys so that existing keys are updated in place instead of duplicated.
@@ -253,12 +368,13 @@ function appendSecretsToFile(resolvedPath, secrets) {
     return;
   }
 
-  let existing = '';
+  const existing = readSecretsFileUtf8OrWarnEmpty(resolvedPath);
   try {
-    const raw = fs.readFileSync(resolvedPath, 'utf8');
-    existing = typeof raw === 'string' ? raw : '';
-  } catch (err) {
-    logger.warn(`Could not read existing secrets file: ${err.message}; appending new keys only.`);
+    yaml.load(existing);
+  } catch {
+    const mergedObj = { ...loadExistingSecrets(resolvedPath), ...secrets };
+    saveSecretsFile(resolvedPath, mergedObj);
+    return;
   }
   const separator = existing.length > 0 && !existing.endsWith('\n') ? '\n' : '';
   fs.writeFileSync(resolvedPath, existing + separator + appendContent, { mode: 0o600 });
@@ -296,7 +412,7 @@ async function generateMissingSecrets(envTemplate, secretsPath) {
 
   appendSecretsToFile(resolvedPath, newSecrets);
 
-  logger.log(`✓ Generated ${missingKeys.length} missing secret key(s): ${missingKeys.join(', ')}`);
+  logger.log(`✔ Generated ${missingKeys.length} missing secret key(s): ${missingKeys.join(', ')}`);
   return missingKeys;
 }
 
@@ -314,6 +430,15 @@ async function generateMissingSecrets(envTemplate, secretsPath) {
  * await createDefaultSecrets('~/.aifabrix/secrets.yaml');
  * // Default secrets file is created
  */
+/** Minimal seed keys for first-time secrets file; values match infra.parameter.yaml generators. */
+const DEFAULT_SECRETS_SEED_KEYS = [
+  'postgres-passwordKeyVault',
+  'redis-passwordKeyVault',
+  'redis-url',
+  'keycloak-admin-passwordKeyVault',
+  'keycloak-web-server-url'
+];
+
 async function createDefaultSecrets(secretsPath) {
   const resolvedPath = secretsPath.startsWith('~')
     ? path.join(os.homedir(), secretsPath.slice(1))
@@ -324,22 +449,17 @@ async function createDefaultSecrets(secretsPath) {
     fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
   }
 
-  const defaultSecrets = `# Local Development Secrets
-# Production uses Azure KeyVault
-
-# Database Secrets
-postgres-passwordKeyVault: "admin123"
-
-# Redis Secrets
-redis-passwordKeyVault: ""
-redis-url: "redis://\${REDIS_HOST}:\${REDIS_PORT}"
+  const seeded = {};
+  for (const k of DEFAULT_SECRETS_SEED_KEYS) {
+    seeded[k] = generateSecretValue(k);
+  }
 
-# Keycloak Secrets
-keycloak-admin-passwordKeyVault: "admin123"
-keycloak-server-url: "http://\${KEYCLOAK_HOST}:\${KEYCLOAK_PORT}"
-`;
+  const header =
+    '# Local development secrets (catalog-aligned seed). Run aifabrix up-infra and aifabrix resolve <app> --force for full keys.\n' +
+    '# Production uses Azure Key Vault.\n\n';
+  const body = yaml.dump(seeded, { indent: 2, lineWidth: -1, noRefs: true, sortKeys: false });
 
-  fs.writeFileSync(resolvedPath, defaultSecrets, { mode: 0o600 });
+  fs.writeFileSync(resolvedPath, header + body, { mode: 0o600 });
 }
 
 module.exports = {