@aifabrix/builder 2.43.0 → 2.44.1

package/lib/internal/fs-real-sync.js

@@ -0,0 +1,104 @@
+/**
+ * Synchronous fs helpers that use `global.__AIFABRIX_NODE_FS_UNMOCKED__` from
+ * `tests/capture-real-fs.js` when present so Jest `jest.mock('fs')` and partial
+ * `jest.mock('internal/node-fs')` cannot break config, schema, or builder scans.
+ *
+ * @fileoverview Real filesystem sync for config, schemas, urls.local registry, url:// resolution
+ */
+'use strict';
+
+/**
+ * @returns {import('node:fs')|null}
+ */
+function unmockedFsSnapshot() {
+  const g = typeof globalThis !== 'undefined' ? globalThis : global;
+  return g && g.__AIFABRIX_NODE_FS_UNMOCKED__ ? g.__AIFABRIX_NODE_FS_UNMOCKED__ : null;
+}
+
+function delegate(name, args) {
+  const snap = unmockedFsSnapshot();
+  if (snap && typeof snap[name] === 'function') {
+    return snap[name](...args);
+  }
+  // When Jest is active, bypass any manual module mocks of node:fs.
+  // This keeps schema/catalog reads stable even if a suite does jest.mock('node:fs').
+  const fs =
+    typeof jest !== 'undefined' && typeof jest.requireActual === 'function'
+      ? jest.requireActual('node:fs')
+      : require('node:fs');
+  return fs[name](...args);
+}
+
+/**
+ * @param {string} p
+ * @returns {boolean}
+ */
+function existsSync(p) {
+  return delegate('existsSync', [p]);
+}
+
+/**
+ * Read file via snapshot/delegate only — not the same binding as {@link readFileSync} on `module.exports`.
+ * Tests may `jest.spyOn(exports, 'readFileSync')`; this function stays the real implementation for env/PIN paths.
+ * @param {string} p
+ * @param {string|import('node:fs').ObjectEncodingOptions} [enc]
+ * @returns {string|Buffer}
+ */
+function readFileSyncDirect(p, enc) {
+  return delegate('readFileSync', enc !== undefined ? [p, enc] : [p]);
+}
+
+/**
+ * @param {string} p
+ * @param {string|import('node:fs').ObjectEncodingOptions} [enc]
+ * @returns {string|Buffer}
+ */
+function readFileSync(p, enc) {
+  return readFileSyncDirect(p, enc);
+}
+
+/**
+ * @param {string} p
+ * @param {string|Buffer} data
+ * @param {object|string} [opts]
+ * @returns {void}
+ */
+function writeFileSync(p, data, opts) {
+  return delegate('writeFileSync', opts !== undefined ? [p, data, opts] : [p, data]);
+}
+
+/**
+ * @param {string} p
+ * @param {object} [opts]
+ * @returns {string|undefined}
+ */
+function mkdirSync(p, opts) {
+  return delegate('mkdirSync', opts !== undefined ? [p, opts] : [p]);
+}
+
+/**
+ * @param {string} p
+ * @returns {import('node:fs').Stats}
+ */
+function statSync(p) {
+  return delegate('statSync', [p]);
+}
+
+/**
+ * @param {string} p
+ * @param {object} [opts]
+ * @returns {string[]|import('node:fs').Dirent[]}
+ */
+function readdirSync(p, opts) {
+  return delegate('readdirSync', opts !== undefined ? [p, opts] : [p]);
+}
+
+module.exports = {
+  existsSync,
+  readFileSync,
+  readFileSyncDirect,
+  writeFileSync,
+  mkdirSync,
+  statSync,
+  readdirSync
+};

package/lib/internal/node-fs.js

@@ -0,0 +1,98 @@
+/**
+ * @fileoverview Filesystem helpers that ignore jest.spyOn on require('node:fs') for sync reads.
+ * Uses jest.requireActual when Jest is active so .bind() captures native implementations even if
+ * this module loads after another suite spied fs. watch still delegates to live fs so tests can spy it.
+ */
+
+'use strict';
+
+/**
+ * Bound native fs from tests/capture-real-fs.js (Jest setupFiles, before mocks).
+ * @returns {Record<string, unknown>|null}
+ */
+function getUnmockedFsSnapshot() {
+  const g = typeof globalThis !== 'undefined' ? globalThis : global;
+  return g && g.__AIFABRIX_NODE_FS_UNMOCKED__ ? g.__AIFABRIX_NODE_FS_UNMOCKED__ : null;
+}
+
+function getRealFs() {
+  const snap = getUnmockedFsSnapshot();
+  if (snap) {
+    return snap;
+  }
+  // When Jest is active, bypass manual module mocks (singleton may still carry spyOn).
+  if (typeof jest !== 'undefined' && typeof jest.requireActual === 'function') {
+    return jest.requireActual('node:fs');
+  }
+  return require('node:fs');
+}
+
+/**
+ * @param {import('node:fs')} fsModule
+ * @param {string} name
+ * @returns {unknown}
+ */
+function bindSync(fsModule, name) {
+  const fn = fsModule[name];
+  return typeof fn === 'function' ? fn.bind(fsModule) : fn;
+}
+
+function liveFs() {
+  return require('node:fs');
+}
+
+/**
+ * Resolve real fs on each access so callers still see the filesystem after other
+ * suites replace require('fs') with mocks (schema-loader, schema sync helpers).
+ * @returns {typeof import('node:fs')}
+ */
+function freshRealFs() {
+  return getRealFs();
+}
+
+/**
+ * @returns {Record<string, unknown>}
+ */
+function buildBoundFs() {
+  const snap = getUnmockedFsSnapshot();
+  const rf = freshRealFs();
+  const live = liveFs();
+  const promiseHost = live.promises || {};
+  const boundPromises = {
+    mkdir: bindSync(promiseHost, 'mkdir'),
+    writeFile: bindSync(promiseHost, 'writeFile'),
+    appendFile: bindSync(promiseHost, 'appendFile'),
+    readFile: bindSync(promiseHost, 'readFile')
+  };
+  if (snap) {
+    return {
+      existsSync: snap.existsSync,
+      readFileSync: snap.readFileSync,
+      writeFileSync: snap.writeFileSync,
+      mkdirSync: snap.mkdirSync,
+      readdirSync: snap.readdirSync,
+      statSync: snap.statSync,
+      watch: (...args) => live.watch(...args),
+      promises: boundPromises
+    };
+  }
+  return {
+    existsSync: bindSync(rf, 'existsSync'),
+    readFileSync: bindSync(rf, 'readFileSync'),
+    writeFileSync: bindSync(rf, 'writeFileSync'),
+    mkdirSync: bindSync(rf, 'mkdirSync'),
+    readdirSync: bindSync(rf, 'readdirSync'),
+    statSync: bindSync(rf, 'statSync'),
+    watch: (...args) => live.watch(...args),
+    promises: boundPromises
+  };
+}
+
+/**
+ * @returns {ReturnType<typeof buildBoundFs>}
+ */
+function nodeFs() {
+  return buildBoundFs();
+}
+
+module.exports = { nodeFs };

package/lib/parameters/database-secret-values.js

@@ -0,0 +1,173 @@
+/**
+ * @fileoverview Index-aware local PostgreSQL URL/password defaults for databases-{app}-{i}-* keys
+ * @author AI Fabrix Team
+ * @version 2.1.0
+ */
+
+const path = require('path');
+const { nodeFs } = require('../internal/node-fs');
+const yaml = require('js-yaml');
+
+/**
+ * Shipped platform templates (when builder/<app> does not exist yet).
+ * Prefer Jest/global.PROJECT_ROOT when that tree contains templates (stable under isolated projects).
+ * @returns {string}
+ */
+function getTemplatesApplicationsRoot() {
+  const fallback = path.join(__dirname, '..', '..', 'templates', 'applications');
+  try {
+    if (typeof global !== 'undefined' && global.PROJECT_ROOT) {
+      const g = path.join(path.resolve(String(global.PROJECT_ROOT)), 'templates', 'applications');
+      const marker = path.join(g, 'miso-controller', 'application.yaml');
+      if (nodeFs().existsSync(marker)) {
+        return g;
+      }
+    }
+  } catch {
+    /* ignore */
+  }
+  return fallback;
+}
+
+/**
+ * PostgreSQL role name from database name (same as compose pgUserName helper).
+ * @param {string} dbName - Logical database name (e.g. miso-logs)
+ * @returns {string}
+ */
+function pgUserFromDbName(dbName) {
+  if (!dbName) return '';
+  return `${String(dbName).replace(/-/g, '_')}_user`;
+}
+
+/**
+ * Local dev password aligned with infra init scripts (user base + _pass123).
+ * @param {string} userName - e.g. miso_user, miso_logs_user
+ * @returns {string}
+ */
+function localDevPasswordFromPgUser(userName) {
+  const base = String(userName).replace(/_user$/i, '');
+  return `${base}_pass123`;
+}
+
+/**
+ * Load requires.databases from application config if present (read-only; no rename).
+ * @param {string} appDir - Absolute path to builder/integration app folder
+ * @returns {Array<{name?: string}>|null}
+ */
+function loadRequiresDatabasesArray(appDir) {
+  if (!appDir || !nodeFs().existsSync(appDir)) return null;
+  const candidates = ['application.yaml', 'application.yml', 'variables.yaml'];
+  for (const name of candidates) {
+    const p = path.join(appDir, name);
+    if (!nodeFs().existsSync(p)) continue;
+    try {
+      const doc = yaml.load(nodeFs().readFileSync(p, 'utf8'));
+      const dbs = doc?.requires?.databases;
+      if (Array.isArray(dbs) && dbs.length > 0) return dbs;
+    } catch {
+      /* ignore */
+    }
+  }
+  return null;
+}
+
+/**
+ * Load requires.databases from shipped Builder template (templates/applications/<appKey>/application.yaml).
+ * @param {string} appKey - Application key
+ * @returns {Array<{name?: string}>|null}
+ */
+function loadShippedRequiresDatabases(appKey) {
+  if (!appKey || typeof appKey !== 'string') return null;
+  const p = path.join(getTemplatesApplicationsRoot(), appKey, 'application.yaml');
+  if (!nodeFs().existsSync(p)) return null;
+  try {
+    const doc = yaml.load(nodeFs().readFileSync(p, 'utf8'));
+    const dbs = doc?.requires?.databases;
+    return Array.isArray(dbs) && dbs.length > 0 ? dbs : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Resolve logical PostgreSQL database name for a databases-{appKey}-{index}-* key.
+ * @param {string} appKey - App key segment from secret key
+ * @param {number} index - Database index
+ * @param {string|null} appDir - Optional app directory to read application.yaml
+ * @returns {string}
+ */
+function resolveLogicalDbName(appKey, index, appDir) {
+  const fromDir = appDir ? loadRequiresDatabasesArray(appDir) : null;
+  const dbs = fromDir || loadShippedRequiresDatabases(appKey);
+  if (dbs && dbs[index] && dbs[index].name) {
+    return String(dbs[index].name);
+  }
+  if (index === 0) {
+    return String(appKey).replace(/-/g, '_');
+  }
+  return `${String(appKey).replace(/-/g, '_')}_${index}`;
+}
+
+/**
+ * Parse databases-{appKey}-{index}-(urlKeyVault|passwordKeyVault).
+ * @param {string} key - Secret key
+ * @returns {{ appKey: string, index: number, kind: 'url'|'password' }|null}
+ */
+function parseDatabaseSecretKey(key) {
+  const m = String(key).match(/^databases-([a-z0-9-]+)-(\d+)-(urlKeyVault|passwordKeyVault)$/i);
+  if (!m) return null;
+  return {
+    appKey: m[1],
+    index: parseInt(m[2], 10),
+    kind: m[3].toLowerCase().startsWith('url') ? 'url' : 'password'
+  };
+}
+
+/**
+ * Build postgres URL with unresolved host/port placeholders.
+ * @param {string} dbName - Database name in connection path (may contain hyphens)
+ * @param {string} userName - DB user
+ * @param {string} password - DB password
+ * @returns {string}
+ */
+function buildPostgresUrlTemplate(dbName, userName, password) {
+  return `postgresql://${userName}:${password}@\${DB_HOST}:\${DB_PORT}/${dbName}`;
+}
+
+/**
+ * Generate password value for databases-*-passwordKeyVault.
+ * @param {string} key - Full secret key
+ * @param {string|null} appDir - Optional app directory for YAML lookup
+ * @returns {string|null} Value or null if key does not match
+ */
+function generateDatabasePasswordValueForKey(key, appDir = null) {
+  const parsed = parseDatabaseSecretKey(key);
+  if (!parsed || parsed.kind !== 'password') return null;
+  const dbName = resolveLogicalDbName(parsed.appKey, parsed.index, appDir);
+  const user = pgUserFromDbName(dbName);
+  return localDevPasswordFromPgUser(user);
+}
+
+/**
+ * Generate URL value for databases-*-urlKeyVault.
+ * @param {string} key - Full secret key
+ * @param {string|null} appDir - Optional app directory for YAML lookup
+ * @returns {string|null} Value or null if key does not match
+ */
+function generateDatabaseUrlValueForKey(key, appDir = null) {
+  const parsed = parseDatabaseSecretKey(key);
+  if (!parsed || parsed.kind !== 'url') return null;
+  const dbName = resolveLogicalDbName(parsed.appKey, parsed.index, appDir);
+  const user = pgUserFromDbName(dbName);
+  const pass = localDevPasswordFromPgUser(user);
+  return buildPostgresUrlTemplate(dbName, user, pass);
+}
+
+module.exports = {
+  parseDatabaseSecretKey,
+  generateDatabasePasswordValueForKey,
+  generateDatabaseUrlValueForKey,
+  loadRequiresDatabasesArray,
+  loadShippedRequiresDatabases,
+  resolveLogicalDbName
+};

package/lib/parameters/infra-kv-discovery.js

@@ -0,0 +1,121 @@
+/**
+ * @fileoverview Discover kv:// keys for up-infra from workspace templates and application.yaml
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+const fsRealSync = require('../internal/fs-real-sync');
+const path = require('path');
+const { loadRequiresDatabasesArray } = require('./database-secret-values');
+
+/**
+ * Extract kv:// secret key names from env template content (active lines only).
+ * @param {string} content - env.template body
+ * @returns {string[]}
+ */
+function extractKvKeysFromEnvContent(content) {
+  if (!content || typeof content !== 'string') return [];
+  const keys = new Set();
+  const kvPattern = /kv:\/\/([a-zA-Z0-9-_]+)/g;
+  const lines = content.split('\n');
+  for (const line of lines) {
+    const t = line.trim();
+    if (t === '' || t.startsWith('#')) continue;
+    let m;
+    kvPattern.lastIndex = 0;
+    while ((m = kvPattern.exec(line)) !== null) {
+      keys.add(m[1]);
+    }
+  }
+  return [...keys];
+}
+
+/**
+ * List app directories for discovery (builder first, then integration-only apps).
+ * @param {object} pathsUtil - paths module
+ * @returns {{ appKey: string, dir: string }[]}
+ */
+function listAppDirsForDiscovery(pathsUtil) {
+  const out = [];
+  const seen = new Set();
+  for (const name of pathsUtil.listBuilderAppNames()) {
+    const dir = pathsUtil.getBuilderPath(name);
+    if (fsRealSync.existsSync(dir)) {
+      out.push({ appKey: name, dir });
+      seen.add(name);
+    }
+  }
+  for (const name of pathsUtil.listIntegrationAppNames()) {
+    if (seen.has(name)) continue;
+    const dir = pathsUtil.getIntegrationPath(name);
+    if (fsRealSync.existsSync(dir)) {
+      out.push({ appKey: name, dir });
+    }
+  }
+  return out;
+}
+
+/**
+ * Derive databases-{appKey}-{i}-url/password keys from requires.databases length.
+ * @param {object} pathsUtil - paths module
+ * @returns {string[]}
+ */
+function deriveDatabaseKvKeysFromWorkspace(pathsUtil) {
+  const keys = new Set();
+  for (const { appKey, dir } of listAppDirsForDiscovery(pathsUtil)) {
+    const dbs = loadRequiresDatabasesArray(dir);
+    if (!Array.isArray(dbs) || dbs.length === 0) continue;
+    for (let i = 0; i < dbs.length; i++) {
+      keys.add(`databases-${appKey}-${i}-urlKeyVault`);
+      keys.add(`databases-${appKey}-${i}-passwordKeyVault`);
+    }
+  }
+  return [...keys];
+}
+
+/**
+ * Keys from env.template files whose catalog entry includes the given hook (e.g. upInfra).
+ * @param {object} pathsUtil - paths module
+ * @param {string} hook - ensureOn hook name
+ * @param {{ keyMatchesEnsureHook: Function }} catalog - loaded catalog API
+ * @returns {string[]}
+ */
+function discoverKvKeysFromEnvTemplatesForHook(pathsUtil, hook, catalog) {
+  const keys = new Set();
+  for (const { dir } of listAppDirsForDiscovery(pathsUtil)) {
+    const envPath = path.join(dir, 'env.template');
+    if (!fsRealSync.existsSync(envPath)) continue;
+    let content;
+    try {
+      content = fsRealSync.readFileSync(envPath, 'utf8');
+    } catch {
+      continue;
+    }
+    for (const k of extractKvKeysFromEnvContent(content)) {
+      if (catalog.keyMatchesEnsureHook(k, hook)) keys.add(k);
+    }
+  }
+  return [...keys];
+}
+
+/**
+ * Full key list for ensureInfraSecrets: catalog exact upInfra + standard miso DB + derived DB + template hooks.
+ * @param {{ getEnsureOnKeys: Function, keyMatchesEnsureHook: Function }} catalog
+ * @param {object} pathsUtil - paths module
+ * @returns {string[]}
+ */
+function getAllInfraEnsureKeys(catalog, pathsUtil) {
+  const set = new Set(catalog.getEnsureOnKeys('upInfra'));
+  for (const k of catalog.getStandardUpInfraBootstrapKeys()) set.add(k);
+  for (const k of deriveDatabaseKvKeysFromWorkspace(pathsUtil)) set.add(k);
+  for (const k of discoverKvKeysFromEnvTemplatesForHook(pathsUtil, 'upInfra', catalog)) set.add(k);
+  return [...set].sort();
+}
+
+module.exports = {
+  extractKvKeysFromEnvContent,
+  deriveDatabaseKvKeysFromWorkspace,
+  discoverKvKeysFromEnvTemplatesForHook,
+  getAllInfraEnsureKeys,
+  listAppDirsForDiscovery
+};