@aifabrix/builder 2.42.1 → 2.44.0

package/lib/infrastructure/services.js

@@ -9,7 +9,6 @@
  */
 
 const { exec } = require('child_process');
-const { promisify } = require('util');
 const path = require('path');
 const fs = require('fs');
 const logger = require('../utils/logger');
@@ -19,13 +18,13 @@ const config = require('../core/config');
 const { getInfraProjectName } = require('./helpers');
 const adminSecrets = require('../core/admin-secrets');
 
-const execAsync = promisify(exec);
-
-// Wrapper to support cwd option
-function execAsyncWithCwd(command, options = {}) {
+// Wrapper to support cwd option and dev-config remote Docker (docker-endpoint + TLS)
+async function execAsyncWithCwd(command, options = {}) {
+  const { getDockerExecEnv } = require('../utils/remote-docker-env');
+  const { cwd, env: extraEnv, ...execOptions } = options;
+  const env = { ...(await getDockerExecEnv()), ...(extraEnv || {}) };
   return new Promise((resolve, reject) => {
-    const { cwd, ...execOptions } = options;
-    exec(command, { ...execOptions, cwd }, (error, stdout, stderr) => {
+    exec(command, { ...execOptions, cwd, env }, (error, stdout, stderr) => {
       if (error) {
         reject(error);
       } else {
@@ -59,14 +58,15 @@ async function startDockerServices(composePath, projectName, adminSecretsPath, i
  * @param {string} pgpassPath - Path to pgpass file
  */
 async function copyPgAdminConfig(pgadminContainerName, serversJsonPath, pgpassPath) {
+  const { execWithDockerEnv } = require('../utils/docker-exec');
   try {
     await new Promise(resolve => setTimeout(resolve, 2000)); // Wait for container to be ready
     if (fs.existsSync(serversJsonPath)) {
-      await execAsync(`docker cp "${serversJsonPath}" ${pgadminContainerName}:/pgadmin4/servers.json`);
+      await execWithDockerEnv(`docker cp "${serversJsonPath}" ${pgadminContainerName}:/pgadmin4/servers.json`);
     }
     if (fs.existsSync(pgpassPath)) {
-      await execAsync(`docker cp "${pgpassPath}" ${pgadminContainerName}:/pgpass`);
-      await execAsync(`docker exec ${pgadminContainerName} chmod 600 /pgpass`);
+      await execWithDockerEnv(`docker cp "${pgpassPath}" ${pgadminContainerName}:/pgpass`);
+      await execWithDockerEnv(`docker exec ${pgadminContainerName} chmod 600 /pgpass`);
     }
   } catch (error) {
     // Ignore copy errors - files might already be there or container not ready
@@ -195,8 +195,10 @@ async function waitForServices(devId = null, opts = {}) {
  * @param {number|string|null} [devId] - Developer ID (null = use current)
  * @param {Object} [options] - Options
  * @param {boolean} [options.strict=false] - When true, only consider current dev's containers (no fallback to dev 0); use for up-miso and status consistency
- * @param {boolean} [options.pgadmin=true] - Include pgAdmin in health check
- * @param {boolean} [options.redisCommander=true] - Include Redis Commander in health check
+ * @param {boolean} [options.postgres=true] - When false, skip Postgres (and pgAdmin) checks — for apps with requires.database: false
+ * @param {boolean} [options.redis=true] - When false, skip Redis (and Redis Commander) checks — for apps with requires.redis: false
+ * @param {boolean} [options.pgadmin=true] - Include pgAdmin in health check (only when Postgres is checked)
+ * @param {boolean} [options.redisCommander=true] - Include Redis Commander in health check (only when Redis is checked)
  * @param {boolean} [options.traefik=false] - Include Traefik in health check
  * @returns {Promise<Object>} Health status of each service
  *
@@ -206,10 +208,14 @@ async function waitForServices(devId = null, opts = {}) {
  */
 async function checkInfraHealth(devId = null, options = {}) {
   const developerId = devId || await config.getDeveloperId();
-  const servicesWithHealthCheck = ['postgres', 'redis'];
+  const includePostgres = options.postgres !== false;
+  const includeRedis = options.redis !== false;
+  const servicesWithHealthCheck = [];
+  if (includePostgres) servicesWithHealthCheck.push('postgres');
+  if (includeRedis) servicesWithHealthCheck.push('redis');
   const servicesWithoutHealthCheck = [];
-  if (options.pgadmin !== false) servicesWithoutHealthCheck.push('pgadmin');
-  if (options.redisCommander !== false) servicesWithoutHealthCheck.push('redis-commander');
+  if (includePostgres && options.pgadmin !== false) servicesWithoutHealthCheck.push('pgadmin');
+  if (includeRedis && options.redisCommander !== false) servicesWithoutHealthCheck.push('redis-commander');
   if (options.traefik === true) servicesWithoutHealthCheck.push('traefik');
   const health = {};
   const lookupOptions = options.strict ? { strict: true } : {};

package/lib/internal/fs-real-sync.js

@@ -0,0 +1,104 @@
+/**
+ * Synchronous fs helpers that use `global.__AIFABRIX_NODE_FS_UNMOCKED__` from
+ * `tests/capture-real-fs.js` when present so Jest `jest.mock('fs')` and partial
+ * `jest.mock('internal/node-fs')` cannot break config, schema, or builder scans.
+ *
+ * @fileoverview Real filesystem sync for config, schemas, urls.local registry, url:// resolution
+ */
+'use strict';
+
+/**
+ * @returns {import('node:fs')|null}
+ */
+function unmockedFsSnapshot() {
+  const g = typeof globalThis !== 'undefined' ? globalThis : global;
+  return g && g.__AIFABRIX_NODE_FS_UNMOCKED__ ? g.__AIFABRIX_NODE_FS_UNMOCKED__ : null;
+}
+
+function delegate(name, args) {
+  const snap = unmockedFsSnapshot();
+  if (snap && typeof snap[name] === 'function') {
+    return snap[name](...args);
+  }
+  // When Jest is active, bypass any manual module mocks of node:fs.
+  // This keeps schema/catalog reads stable even if a suite does jest.mock('node:fs').
+  const fs =
+    typeof jest !== 'undefined' && typeof jest.requireActual === 'function'
+      ? jest.requireActual('node:fs')
+      : require('node:fs');
+  return fs[name](...args);
+}
+
+/**
+ * @param {string} p
+ * @returns {boolean}
+ */
+function existsSync(p) {
+  return delegate('existsSync', [p]);
+}
+
+/**
+ * Read file via snapshot/delegate only — not the same binding as {@link readFileSync} on `module.exports`.
+ * Tests may `jest.spyOn(exports, 'readFileSync')`; this function stays the real implementation for env/PIN paths.
+ * @param {string} p
+ * @param {string|import('node:fs').ObjectEncodingOptions} [enc]
+ * @returns {string|Buffer}
+ */
+function readFileSyncDirect(p, enc) {
+  return delegate('readFileSync', enc !== undefined ? [p, enc] : [p]);
+}
+
+/**
+ * @param {string} p
+ * @param {string|import('node:fs').ObjectEncodingOptions} [enc]
+ * @returns {string|Buffer}
+ */
+function readFileSync(p, enc) {
+  return readFileSyncDirect(p, enc);
+}
+
+/**
+ * @param {string} p
+ * @param {string|Buffer} data
+ * @param {object|string} [opts]
+ * @returns {void}
+ */
+function writeFileSync(p, data, opts) {
+  return delegate('writeFileSync', opts !== undefined ? [p, data, opts] : [p, data]);
+}
+
+/**
+ * @param {string} p
+ * @param {object} [opts]
+ * @returns {string|undefined}
+ */
+function mkdirSync(p, opts) {
+  return delegate('mkdirSync', opts !== undefined ? [p, opts] : [p]);
+}
+
+/**
+ * @param {string} p
+ * @returns {import('node:fs').Stats}
+ */
+function statSync(p) {
+  return delegate('statSync', [p]);
+}
+
+/**
+ * @param {string} p
+ * @param {object} [opts]
+ * @returns {string[]|import('node:fs').Dirent[]}
+ */
+function readdirSync(p, opts) {
+  return delegate('readdirSync', opts !== undefined ? [p, opts] : [p]);
+}
+
+module.exports = {
+  existsSync,
+  readFileSync,
+  readFileSyncDirect,
+  writeFileSync,
+  mkdirSync,
+  statSync,
+  readdirSync
+};

package/lib/internal/node-fs.js

@@ -0,0 +1,98 @@
+/**
+ * @fileoverview Filesystem helpers that ignore jest.spyOn on require('node:fs') for sync reads.
+ * Uses jest.requireActual when Jest is active so .bind() captures native implementations even if
+ * this module loads after another suite spied fs. watch still delegates to live fs so tests can spy it.
+ */
+
+'use strict';
+
+/**
+ * Bound native fs from tests/capture-real-fs.js (Jest setupFiles, before mocks).
+ * @returns {Record<string, unknown>|null}
+ */
+function getUnmockedFsSnapshot() {
+  const g = typeof globalThis !== 'undefined' ? globalThis : global;
+  return g && g.__AIFABRIX_NODE_FS_UNMOCKED__ ? g.__AIFABRIX_NODE_FS_UNMOCKED__ : null;
+}
+
+function getRealFs() {
+  const snap = getUnmockedFsSnapshot();
+  if (snap) {
+    return snap;
+  }
+  // When Jest is active, bypass manual module mocks (singleton may still carry spyOn).
+  if (typeof jest !== 'undefined' && typeof jest.requireActual === 'function') {
+    return jest.requireActual('node:fs');
+  }
+  return require('node:fs');
+}
+
+/**
+ * @param {import('node:fs')} fsModule
+ * @param {string} name
+ * @returns {unknown}
+ */
+function bindSync(fsModule, name) {
+  const fn = fsModule[name];
+  return typeof fn === 'function' ? fn.bind(fsModule) : fn;
+}
+
+function liveFs() {
+  return require('node:fs');
+}
+
+/**
+ * Resolve real fs on each access so callers still see the filesystem after other
+ * suites replace require('fs') with mocks (schema-loader, schema sync helpers).
+ * @returns {typeof import('node:fs')}
+ */
+function freshRealFs() {
+  return getRealFs();
+}
+
+/**
+ * @returns {Record<string, unknown>}
+ */
+function buildBoundFs() {
+  const snap = getUnmockedFsSnapshot();
+  const rf = freshRealFs();
+  const live = liveFs();
+  const promiseHost = live.promises || {};
+  const boundPromises = {
+    mkdir: bindSync(promiseHost, 'mkdir'),
+    writeFile: bindSync(promiseHost, 'writeFile'),
+    appendFile: bindSync(promiseHost, 'appendFile'),
+    readFile: bindSync(promiseHost, 'readFile')
+  };
+  if (snap) {
+    return {
+      existsSync: snap.existsSync,
+      readFileSync: snap.readFileSync,
+      writeFileSync: snap.writeFileSync,
+      mkdirSync: snap.mkdirSync,
+      readdirSync: snap.readdirSync,
+      statSync: snap.statSync,
+      watch: (...args) => live.watch(...args),
+      promises: boundPromises
+    };
+  }
+  return {
+    existsSync: bindSync(rf, 'existsSync'),
+    readFileSync: bindSync(rf, 'readFileSync'),
+    writeFileSync: bindSync(rf, 'writeFileSync'),
+    mkdirSync: bindSync(rf, 'mkdirSync'),
+    readdirSync: bindSync(rf, 'readdirSync'),
+    statSync: bindSync(rf, 'statSync'),
+    watch: (...args) => live.watch(...args),
+    promises: boundPromises
+  };
+}
+
+/**
+ * @returns {ReturnType<typeof buildBoundFs>}
+ */
+function nodeFs() {
+  return buildBoundFs();
+}
+
+module.exports = { nodeFs };

package/lib/parameters/database-secret-values.js

@@ -0,0 +1,173 @@
+/**
+ * @fileoverview Index-aware local PostgreSQL URL/password defaults for databases-{app}-{i}-* keys
+ * @author AI Fabrix Team
+ * @version 2.1.0
+ */
+
+const path = require('path');
+const { nodeFs } = require('../internal/node-fs');
+const yaml = require('js-yaml');
+
+/**
+ * Shipped platform templates (when builder/<app> does not exist yet).
+ * Prefer Jest/global.PROJECT_ROOT when that tree contains templates (stable under isolated projects).
+ * @returns {string}
+ */
+function getTemplatesApplicationsRoot() {
+  const fallback = path.join(__dirname, '..', '..', 'templates', 'applications');
+  try {
+    if (typeof global !== 'undefined' && global.PROJECT_ROOT) {
+      const g = path.join(path.resolve(String(global.PROJECT_ROOT)), 'templates', 'applications');
+      const marker = path.join(g, 'miso-controller', 'application.yaml');
+      if (nodeFs().existsSync(marker)) {
+        return g;
+      }
+    }
+  } catch {
+    /* ignore */
+  }
+  return fallback;
+}
+
+/**
+ * PostgreSQL role name from database name (same as compose pgUserName helper).
+ * @param {string} dbName - Logical database name (e.g. miso-logs)
+ * @returns {string}
+ */
+function pgUserFromDbName(dbName) {
+  if (!dbName) return '';
+  return `${String(dbName).replace(/-/g, '_')}_user`;
+}
+
+/**
+ * Local dev password aligned with infra init scripts (user base + _pass123).
+ * @param {string} userName - e.g. miso_user, miso_logs_user
+ * @returns {string}
+ */
+function localDevPasswordFromPgUser(userName) {
+  const base = String(userName).replace(/_user$/i, '');
+  return `${base}_pass123`;
+}
+
+/**
+ * Load requires.databases from application config if present (read-only; no rename).
+ * @param {string} appDir - Absolute path to builder/integration app folder
+ * @returns {Array<{name?: string}>|null}
+ */
+function loadRequiresDatabasesArray(appDir) {
+  if (!appDir || !nodeFs().existsSync(appDir)) return null;
+  const candidates = ['application.yaml', 'application.yml', 'variables.yaml'];
+  for (const name of candidates) {
+    const p = path.join(appDir, name);
+    if (!nodeFs().existsSync(p)) continue;
+    try {
+      const doc = yaml.load(nodeFs().readFileSync(p, 'utf8'));
+      const dbs = doc?.requires?.databases;
+      if (Array.isArray(dbs) && dbs.length > 0) return dbs;
+    } catch {
+      /* ignore */
+    }
+  }
+  return null;
+}
+
+/**
+ * Load requires.databases from shipped Builder template (templates/applications/<appKey>/application.yaml).
+ * @param {string} appKey - Application key
+ * @returns {Array<{name?: string}>|null}
+ */
+function loadShippedRequiresDatabases(appKey) {
+  if (!appKey || typeof appKey !== 'string') return null;
+  const p = path.join(getTemplatesApplicationsRoot(), appKey, 'application.yaml');
+  if (!nodeFs().existsSync(p)) return null;
+  try {
+    const doc = yaml.load(nodeFs().readFileSync(p, 'utf8'));
+    const dbs = doc?.requires?.databases;
+    return Array.isArray(dbs) && dbs.length > 0 ? dbs : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Resolve logical PostgreSQL database name for a databases-{appKey}-{index}-* key.
+ * @param {string} appKey - App key segment from secret key
+ * @param {number} index - Database index
+ * @param {string|null} appDir - Optional app directory to read application.yaml
+ * @returns {string}
+ */
+function resolveLogicalDbName(appKey, index, appDir) {
+  const fromDir = appDir ? loadRequiresDatabasesArray(appDir) : null;
+  const dbs = fromDir || loadShippedRequiresDatabases(appKey);
+  if (dbs && dbs[index] && dbs[index].name) {
+    return String(dbs[index].name);
+  }
+  if (index === 0) {
+    return String(appKey).replace(/-/g, '_');
+  }
+  return `${String(appKey).replace(/-/g, '_')}_${index}`;
+}
+
+/**
+ * Parse databases-{appKey}-{index}-(urlKeyVault|passwordKeyVault).
+ * @param {string} key - Secret key
+ * @returns {{ appKey: string, index: number, kind: 'url'|'password' }|null}
+ */
+function parseDatabaseSecretKey(key) {
+  const m = String(key).match(/^databases-([a-z0-9-]+)-(\d+)-(urlKeyVault|passwordKeyVault)$/i);
+  if (!m) return null;
+  return {
+    appKey: m[1],
+    index: parseInt(m[2], 10),
+    kind: m[3].toLowerCase().startsWith('url') ? 'url' : 'password'
+  };
+}
+
+/**
+ * Build postgres URL with unresolved host/port placeholders.
+ * @param {string} dbName - Database name in connection path (may contain hyphens)
+ * @param {string} userName - DB user
+ * @param {string} password - DB password
+ * @returns {string}
+ */
+function buildPostgresUrlTemplate(dbName, userName, password) {
+  return `postgresql://${userName}:${password}@\${DB_HOST}:\${DB_PORT}/${dbName}`;
+}
+
+/**
+ * Generate password value for databases-*-passwordKeyVault.
+ * @param {string} key - Full secret key
+ * @param {string|null} appDir - Optional app directory for YAML lookup
+ * @returns {string|null} Value or null if key does not match
+ */
+function generateDatabasePasswordValueForKey(key, appDir = null) {
+  const parsed = parseDatabaseSecretKey(key);
+  if (!parsed || parsed.kind !== 'password') return null;
+  const dbName = resolveLogicalDbName(parsed.appKey, parsed.index, appDir);
+  const user = pgUserFromDbName(dbName);
+  return localDevPasswordFromPgUser(user);
+}
+
+/**
+ * Generate URL value for databases-*-urlKeyVault.
+ * @param {string} key - Full secret key
+ * @param {string|null} appDir - Optional app directory for YAML lookup
+ * @returns {string|null} Value or null if key does not match
+ */
+function generateDatabaseUrlValueForKey(key, appDir = null) {
+  const parsed = parseDatabaseSecretKey(key);
+  if (!parsed || parsed.kind !== 'url') return null;
+  const dbName = resolveLogicalDbName(parsed.appKey, parsed.index, appDir);
+  const user = pgUserFromDbName(dbName);
+  const pass = localDevPasswordFromPgUser(user);
+  return buildPostgresUrlTemplate(dbName, user, pass);
+}
+
+module.exports = {
+  parseDatabaseSecretKey,
+  generateDatabasePasswordValueForKey,
+  generateDatabaseUrlValueForKey,
+  loadRequiresDatabasesArray,
+  loadShippedRequiresDatabases,
+  resolveLogicalDbName
+};

package/lib/parameters/infra-kv-discovery.js

@@ -0,0 +1,121 @@
+/**
+ * @fileoverview Discover kv:// keys for up-infra from workspace templates and application.yaml
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fsRealSync = require('../internal/fs-real-sync');
+const path = require('path');
+const { loadRequiresDatabasesArray } = require('./database-secret-values');
+
+/**
+ * Extract kv:// secret key names from env template content (active lines only).
+ * @param {string} content - env.template body
+ * @returns {string[]}
+ */
+function extractKvKeysFromEnvContent(content) {
+  if (!content || typeof content !== 'string') return [];
+  const keys = new Set();
+  const kvPattern = /kv:\/\/([a-zA-Z0-9-_]+)/g;
+  const lines = content.split('\n');
+  for (const line of lines) {
+    const t = line.trim();
+    if (t === '' || t.startsWith('#')) continue;
+    let m;
+    kvPattern.lastIndex = 0;
+    while ((m = kvPattern.exec(line)) !== null) {
+      keys.add(m[1]);
+    }
+  }
+  return [...keys];
+}
+
+/**
+ * List app directories for discovery (builder first, then integration-only apps).
+ * @param {object} pathsUtil - paths module
+ * @returns {{ appKey: string, dir: string }[]}
+ */
+function listAppDirsForDiscovery(pathsUtil) {
+  const out = [];
+  const seen = new Set();
+  for (const name of pathsUtil.listBuilderAppNames()) {
+    const dir = pathsUtil.getBuilderPath(name);
+    if (fsRealSync.existsSync(dir)) {
+      out.push({ appKey: name, dir });
+      seen.add(name);
+    }
+  }
+  for (const name of pathsUtil.listIntegrationAppNames()) {
+    if (seen.has(name)) continue;
+    const dir = pathsUtil.getIntegrationPath(name);
+    if (fsRealSync.existsSync(dir)) {
+      out.push({ appKey: name, dir });
+    }
+  }
+  return out;
+}
+
+/**
+ * Derive databases-{appKey}-{i}-url/password keys from requires.databases length.
+ * @param {object} pathsUtil - paths module
+ * @returns {string[]}
+ */
+function deriveDatabaseKvKeysFromWorkspace(pathsUtil) {
+  const keys = new Set();
+  for (const { appKey, dir } of listAppDirsForDiscovery(pathsUtil)) {
+    const dbs = loadRequiresDatabasesArray(dir);
+    if (!Array.isArray(dbs) || dbs.length === 0) continue;
+    for (let i = 0; i < dbs.length; i++) {
+      keys.add(`databases-${appKey}-${i}-urlKeyVault`);
+      keys.add(`databases-${appKey}-${i}-passwordKeyVault`);
+    }
+  }
+  return [...keys];
+}
+
+/**
+ * Keys from env.template files whose catalog entry includes the given hook (e.g. upInfra).
+ * @param {object} pathsUtil - paths module
+ * @param {string} hook - ensureOn hook name
+ * @param {{ keyMatchesEnsureHook: Function }} catalog - loaded catalog API
+ * @returns {string[]}
+ */
+function discoverKvKeysFromEnvTemplatesForHook(pathsUtil, hook, catalog) {
+  const keys = new Set();
+  for (const { dir } of listAppDirsForDiscovery(pathsUtil)) {
+    const envPath = path.join(dir, 'env.template');
+    if (!fsRealSync.existsSync(envPath)) continue;
+    let content;
+    try {
+      content = fsRealSync.readFileSync(envPath, 'utf8');
+    } catch {
+      continue;
+    }
+    for (const k of extractKvKeysFromEnvContent(content)) {
+      if (catalog.keyMatchesEnsureHook(k, hook)) keys.add(k);
+    }
+  }
+  return [...keys];
+}
+
+/**
+ * Full key list for ensureInfraSecrets: catalog exact upInfra + standard miso DB + derived DB + template hooks.
+ * @param {{ getEnsureOnKeys: Function, keyMatchesEnsureHook: Function }} catalog
+ * @param {object} pathsUtil - paths module
+ * @returns {string[]}
+ */
+function getAllInfraEnsureKeys(catalog, pathsUtil) {
+  const set = new Set(catalog.getEnsureOnKeys('upInfra'));
+  for (const k of catalog.getStandardUpInfraBootstrapKeys()) set.add(k);
+  for (const k of deriveDatabaseKvKeysFromWorkspace(pathsUtil)) set.add(k);
+  for (const k of discoverKvKeysFromEnvTemplatesForHook(pathsUtil, 'upInfra', catalog)) set.add(k);
+  return [...set].sort();
+}
+
+module.exports = {
+  extractKvKeysFromEnvContent,
+  deriveDatabaseKvKeysFromWorkspace,
+  discoverKvKeysFromEnvTemplatesForHook,
+  getAllInfraEnsureKeys,
+  listAppDirsForDiscovery
+};