@aifabrix/builder 2.44.4 → 2.44.6

package/lib/infrastructure/index.js

@@ -26,6 +26,7 @@ const {
   registerHandlebarsHelper
 } = require('./helpers');
 const secretsEnsure = require('../core/secrets-ensure');
+const { getRestartableInfraServiceNames } = require('../constants/infra-compose-service-names');
 const {
   buildTraefikConfig,
   validateTraefikConfig,
@@ -36,6 +37,7 @@ const {
   startDockerServicesAndConfigure,
   checkInfraHealth
 } = require('./services');
+const { tryComposeProjectDown, stopInfraDockerStackOrphaned } = require('./orphan-infra-docker-teardown');
 const adminSecrets = require('../core/admin-secrets');
 // Lazy require to avoid circular dependency: infra -> app/down -> run-helpers -> infra
 
@@ -220,9 +222,18 @@ async function stopInfra() {
   const devId = await config.getDeveloperId();
   const { infraDir, adminSecretsPath } = resolveInfraStatePaths(devId);
   const composePath = path.join(infraDir, 'compose.yaml');
+  const hasCompose = fs.existsSync(composePath);
+  const hasAdminSecrets = fs.existsSync(adminSecretsPath);
 
-  if (!fs.existsSync(composePath) || !fs.existsSync(adminSecretsPath)) {
-    logger.log('Infrastructure not running or not properly configured');
+  if (!hasCompose || !hasAdminSecrets) {
+    logger.log('Infra compose or admin-secrets missing; stopping Docker stack (no volume delete)...');
+    await stopAllAppContainers(devId);
+    try {
+      await tryComposeProjectDown(devId, false);
+    } catch (err) {
+      logger.log(`Compose project down failed (${err.message}); applying orphan cleanup...`);
+      await stopInfraDockerStackOrphaned(devId, { removeVolumes: false });
+    }
     return;
   }
 
@@ -281,21 +292,34 @@ async function stopInfraWithVolumes() {
   const devId = await config.getDeveloperId();
   const { infraDir, adminSecretsPath } = resolveInfraStatePaths(devId);
   const composePath = path.join(infraDir, 'compose.yaml');
+  const hasCompose = fs.existsSync(composePath);
+  const hasAdminSecrets = fs.existsSync(adminSecretsPath);
 
-  if (!fs.existsSync(composePath) || !fs.existsSync(adminSecretsPath)) {
-    logger.log('Infrastructure not running or not properly configured');
+  await stopAllAppContainersAndVolumes(devId);
+
+  if (hasCompose && hasAdminSecrets) {
+    await withRunEnv(infraDir, adminSecretsPath, async(runEnvPath) => {
+      logger.log('Stopping infrastructure services and removing all data...');
+      const projectName = getInfraProjectName(devId);
+      const composeCmd = await dockerUtils.getComposeCommand();
+      try {
+        await execAsyncWithCwd(`${composeCmd} -f "${composePath}" -p ${projectName} --env-file "${runEnvPath}" down -v`, { cwd: infraDir });
+      } catch (err) {
+        logger.log(`Compose down failed (${err.message}); applying orphan cleanup...`);
+        await stopInfraDockerStackOrphaned(devId, { removeVolumes: true });
+      }
+      logger.log('Infrastructure services stopped and all data removed');
+    });
     return;
   }
 
-  await withRunEnv(infraDir, adminSecretsPath, async(runEnvPath) => {
-    logger.log('Stopping application containers on the same network...');
-    await stopAllAppContainersAndVolumes(devId);
-    logger.log('Stopping infrastructure services and removing all data...');
-    const projectName = getInfraProjectName(devId);
-    const composeCmd = await dockerUtils.getComposeCommand();
-    await execAsyncWithCwd(`${composeCmd} -f "${composePath}" -p ${projectName} --env-file "${runEnvPath}" down -v`, { cwd: infraDir });
-    logger.log('Infrastructure services stopped and all data removed');
-  });
+  logger.log('Infra compose or admin-secrets missing; tearing down stuck Docker stack (project/volumes/network)...');
+  try {
+    await tryComposeProjectDown(devId, true);
+  } catch (err) {
+    logger.log(`Compose project down failed (${err.message}); applying orphan cleanup...`);
+    await stopInfraDockerStackOrphaned(devId, { removeVolumes: true });
+  }
 }
 
 /**
@@ -305,18 +329,19 @@ async function stopInfraWithVolumes() {
  * @async
  * @function restartService
  * @param {string} serviceName - Name of service to restart
+ * @param {{ suppressProgressLog?: boolean }} [options] - When `suppressProgressLog: true`, skip internal logger lines (CLI prints layout).
  * @returns {Promise<void>} Resolves when service is restarted
  * @throws {Error} If service doesn't exist or restart fails
  *
  * @example
- * await restartService('keycloak');
- * // Keycloak service is restarted
+ * await restartService('postgres');
+ * // Postgres service is restarted
  */
-async function restartService(serviceName) {
+async function restartService(serviceName, options = {}) {
   if (!serviceName || typeof serviceName !== 'string') {
     throw new Error('Service name is required and must be a string');
   }
-  const validServices = ['postgres', 'redis', 'pgadmin', 'redis-commander', 'traefik'];
+  const validServices = getRestartableInfraServiceNames();
   if (!validServices.includes(serviceName)) {
     throw new Error(`Invalid service name. Must be one of: ${validServices.join(', ')}`);
   }
@@ -329,12 +354,17 @@ async function restartService(serviceName) {
     throw new Error('Infrastructure not properly configured');
   }
 
+  const suppressLog = Boolean(options && options.suppressProgressLog);
   await withRunEnv(infraDir, adminSecretsPath, async(runEnvPath) => {
-    logger.log(`Restarting ${serviceName} service...`);
+    if (!suppressLog) {
+      logger.log(`Restarting ${serviceName} service...`);
+    }
     const projectName = getInfraProjectName(devId);
     const composeCmd = await dockerUtils.getComposeCommand();
     await execAsyncWithCwd(`${composeCmd} -f "${composePath}" -p ${projectName} --env-file "${runEnvPath}" restart ${serviceName}`, { cwd: infraDir });
-    logger.log(`${serviceName} service restarted successfully`);
+    if (!suppressLog) {
+      logger.log(`${serviceName} service restarted successfully`);
+    }
   });
 }
 

package/lib/infrastructure/orphan-infra-docker-teardown.js

@@ -0,0 +1,177 @@
+/**
+ * Tear down AI Fabrix infra Docker resources when compose.yaml / admin-secrets
+ * are missing from disk (e.g. after manual deletes). Matches compose naming from
+ * `templates/infra/compose.yaml.hbs` and `lib/infrastructure/compose.js`.
+ *
+ * @fileoverview Orphan Docker teardown for developer-scoped infra stacks
+ */
+
+'use strict';
+
+const dockerUtils = require('../utils/docker');
+const logger = require('../utils/logger');
+const { execAsyncWithCwd } = require('./services');
+const { getInfraProjectName } = require('./helpers');
+
+/**
+ * Bridge network name for this developer (must match compose generator).
+ * @param {string|number} devId - Developer ID
+ * @returns {string}
+ */
+function getInfraBridgeNetworkName(devId) {
+  const idNum = typeof devId === 'string' ? parseInt(devId, 10) : devId;
+  return idNum === 0 ? 'infra-aifabrix-network' : `infra-dev${devId}-aifabrix-network`;
+}
+
+/**
+ * Known infra service container names (postgres, redis, optional UIs).
+ * @param {string|number} devId - Developer ID
+ * @returns {string[]}
+ */
+function getInfraServiceContainerNames(devId) {
+  const idNum = typeof devId === 'string' ? parseInt(devId, 10) : devId;
+  if (idNum === 0) {
+    return ['aifabrix-postgres', 'aifabrix-redis', 'aifabrix-pgadmin', 'aifabrix-redis-commander', 'aifabrix-traefik'];
+  }
+  return [
+    `aifabrix-dev${devId}-postgres`,
+    `aifabrix-dev${devId}-redis`,
+    `aifabrix-dev${devId}-pgadmin`,
+    `aifabrix-dev${devId}-redis-commander`,
+    `aifabrix-dev${devId}-traefik`
+  ];
+}
+
+/**
+ * Named volumes declared in infra compose (explicit `name:` entries).
+ * @param {string|number} devId - Developer ID
+ * @returns {string[]}
+ */
+function getInfraNamedVolumeCandidates(devId) {
+  const idNum = typeof devId === 'string' ? parseInt(devId, 10) : devId;
+  if (idNum === 0) {
+    return ['infra_postgres_data', 'infra_redis_data', 'infra_pgadmin_data'];
+  }
+  return [
+    `infra_dev${devId}_postgres_data`,
+    `infra_dev${devId}_redis_data`,
+    `infra_dev${devId}_pgadmin_data`
+  ];
+}
+
+/**
+ * `docker compose down` using only the Compose project name (works when the
+ * compose file was removed but the project still exists in Docker).
+ *
+ * @param {string|number} devId - Developer ID
+ * @param {boolean} withVolumes - Pass `-v` to remove named volumes
+ * @returns {Promise<void>}
+ */
+async function tryComposeProjectDown(devId, withVolumes) {
+  const composeCmd = await dockerUtils.getComposeCommand();
+  const projectName = getInfraProjectName(devId);
+  const volFlag = withVolumes ? ' -v' : '';
+  await execAsyncWithCwd(`${composeCmd} -p "${projectName}" down${volFlag}`);
+}
+
+/**
+ * @param {string} name - Container name
+ * @returns {Promise<void>}
+ */
+async function dockerRmForceOne(name) {
+  try {
+    await execAsyncWithCwd(`docker rm -f "${name}"`);
+    logger.log(`Stopped and removed container: ${name}`);
+  } catch {
+    logger.log(`Container ${name} not running or already removed`);
+  }
+}
+
+/**
+ * @param {string} vol - Volume name
+ * @returns {Promise<void>}
+ */
+async function dockerVolumeRmForceOne(vol) {
+  try {
+    await execAsyncWithCwd(`docker volume rm -f "${vol}"`);
+    logger.log(`Removed volume: ${vol}`);
+  } catch {
+    logger.log(`Volume ${vol} not found or already removed`);
+  }
+}
+
+/**
+ * Remove containers on the developer infra bridge network (covers strays).
+ * @param {string} networkName - Docker network name
+ * @returns {Promise<void>}
+ */
+async function removeContainersOnNetwork(networkName) {
+  let stdout = '';
+  try {
+    ({ stdout } = await execAsyncWithCwd(
+      `docker network inspect "${networkName}" --format '{{json .Containers}}'`
+    ));
+  } catch {
+    return;
+  }
+  const raw = String(stdout || '').trim();
+  if (!raw || raw === 'null' || raw === '{}') return;
+  let containers;
+  try {
+    containers = JSON.parse(raw);
+  } catch {
+    return;
+  }
+  for (const endpoint of Object.values(containers)) {
+    const name = endpoint && typeof endpoint.Name === 'string' ? endpoint.Name.replace(/^\//, '') : '';
+    if (name) await dockerRmForceOne(name);
+  }
+}
+
+/**
+ * Remove bridge network if present.
+ * @param {string} networkName - Docker network name
+ * @returns {Promise<void>}
+ */
+async function removeNetworkIfPresent(networkName) {
+  try {
+    await execAsyncWithCwd(`docker network rm "${networkName}"`);
+    logger.log(`Removed network: ${networkName}`);
+  } catch {
+    logger.log(`Network ${networkName} not removed (still in use or already gone)`);
+  }
+}
+
+/**
+ * Last-resort teardown: remove known infra containers, optional volumes, then network.
+ *
+ * @param {string|number} devId - Developer ID
+ * @param {{ removeVolumes?: boolean }} [opts]
+ * @returns {Promise<void>}
+ */
+async function stopInfraDockerStackOrphaned(devId, opts = {}) {
+  const removeVolumes = opts.removeVolumes !== false;
+  const networkName = getInfraBridgeNetworkName(devId);
+
+  for (const name of getInfraServiceContainerNames(devId)) {
+    await dockerRmForceOne(name);
+  }
+
+  await removeContainersOnNetwork(networkName);
+
+  if (removeVolumes) {
+    for (const vol of getInfraNamedVolumeCandidates(devId)) {
+      await dockerVolumeRmForceOne(vol);
+    }
+  }
+
+  await removeNetworkIfPresent(networkName);
+}
+
+module.exports = {
+  getInfraBridgeNetworkName,
+  getInfraServiceContainerNames,
+  getInfraNamedVolumeCandidates,
+  tryComposeProjectDown,
+  stopInfraDockerStackOrphaned
+};

package/lib/parameters/infra-kv-discovery.js

@@ -30,6 +30,23 @@ function extractKvKeysFromEnvContent(content) {
   return [...keys];
 }
 
+/**
+ * List builder app directories only (no integration/* apps).
+ * Used by `parameters validate` so sample integrations do not require catalog entries.
+ * @param {object} pathsUtil - paths module
+ * @returns {{ appKey: string, dir: string }[]}
+ */
+function listBuilderAppDirsForDiscovery(pathsUtil) {
+  const out = [];
+  for (const name of pathsUtil.listBuilderAppNames()) {
+    const dir = pathsUtil.getBuilderPath(name);
+    if (fsRealSync.existsSync(dir)) {
+      out.push({ appKey: name, dir });
+    }
+  }
+  return out;
+}
+
 /**
  * List app directories for discovery (builder first, then integration-only apps).
  * @param {object} pathsUtil - paths module
@@ -99,13 +116,20 @@ function discoverKvKeysFromEnvTemplatesForHook(pathsUtil, hook, catalog) {
 }
 
 /**
- * Full key list for ensureInfraSecrets: catalog exact upInfra + standard miso DB + derived DB + template hooks.
- * @param {{ getEnsureOnKeys: Function, keyMatchesEnsureHook: Function }} catalog
+ * Keys `ensureInfraSecrets` (`up-infra`) may write locally:
+ * - {@link standardUpInfraEnsureKeys} in infra.parameter.yaml (explicit bootstrap + Redis/Postgres core)
+ * - `databases-{app}-{i}-*` derived from each workspace app `requires.databases`
+ * - `kv://` names from env.template lines whose catalog entry includes `upInfra`
+ *
+ * Does **not** union every `parameters[].ensureOn: upInfra` catalog entry—those would create dozens of
+ * unused Azure/BASH/app placeholders (often emptyString locally). Apps pull those via `resolve` / run when needed.
+ *
+ * @param {{ getStandardUpInfraBootstrapKeys: Function, keyMatchesEnsureHook: Function }} catalog
  * @param {object} pathsUtil - paths module
  * @returns {string[]}
  */
 function getAllInfraEnsureKeys(catalog, pathsUtil) {
-  const set = new Set(catalog.getEnsureOnKeys('upInfra'));
+  const set = new Set();
   for (const k of catalog.getStandardUpInfraBootstrapKeys()) set.add(k);
   for (const k of deriveDatabaseKvKeysFromWorkspace(pathsUtil)) set.add(k);
   for (const k of discoverKvKeysFromEnvTemplatesForHook(pathsUtil, 'upInfra', catalog)) set.add(k);
@@ -117,5 +141,6 @@ module.exports = {
   deriveDatabaseKvKeysFromWorkspace,
   discoverKvKeysFromEnvTemplatesForHook,
   getAllInfraEnsureKeys,
-  listAppDirsForDiscovery
+  listAppDirsForDiscovery,
+  listBuilderAppDirsForDiscovery
 };

package/lib/parameters/infra-parameter-catalog.js

@@ -133,7 +133,10 @@ function createCatalogApi(doc, exact, patterns) {
 
   function isKeyAllowedEmpty(key) {
     const entry = findEntryForKey(key);
-    return Boolean(entry && entry.generator && entry.generator.type === 'emptyAllowed');
+    const gen = entry && entry.generator;
+    // emptyAllowed: e.g. Redis password when no requirepass.
+    // emptyString: optional user-supplied keys (OpenAI / Azure OpenAI) — absent resolves like ''.
+    return Boolean(gen && (gen.type === 'emptyAllowed' || gen.type === 'emptyString'));
   }
 
   function keyMatchesEnsureHook(key, hook) {
@@ -236,7 +239,7 @@ function readRelaxedUpInfraEnsureKeyList(catalogPath = DEFAULT_CATALOG_PATH) {
 }
 
 /**
- * YAML-only: keys whose generator type is emptyAllowed (for ensure backfill behavior).
+ * YAML-only: keys whose generator type is emptyAllowed or emptyString (optional / absent OK).
  *
  * @param {string} [catalogPath]
  * @returns {Set<string>|null}
@@ -254,7 +257,7 @@ function readRelaxedEmptyAllowedKeySet(catalogPath = DEFAULT_CATALOG_PATH) {
         typeof entry.key === 'string' &&
         entry.key.trim() &&
         entry.generator &&
-        entry.generator.type === 'emptyAllowed'
+        (entry.generator.type === 'emptyAllowed' || entry.generator.type === 'emptyString')
       ) {
         set.add(entry.key.trim());
       }

package/lib/parameters/infra-parameter-validate.js

@@ -6,38 +6,86 @@
 
 const { nodeFs } = require('../internal/node-fs');
 const path = require('path');
-const { listAppDirsForDiscovery, extractKvKeysFromEnvContent } = require('./infra-kv-discovery');
+const {
+  listBuilderAppDirsForDiscovery,
+  extractKvKeysFromEnvContent
+} = require('./infra-kv-discovery');
 
-/**
- * Validate that every kv:// key in workspace env.template files has catalog coverage.
- * @param {{ findEntryForKey: Function }} catalog - Loaded catalog API
- * @param {object} pathsUtil - paths module
- * @returns {{ valid: boolean, errors: string[] }}
- */
-function validateWorkspaceKvRefsAgainstCatalog(catalog, pathsUtil) {
-  const errors = [];
+function _scanEnvTemplate(fs, cwd, envPath) {
+  const rel = path.relative(cwd, envPath) || envPath;
+  try {
+    const content = fs.readFileSync(envPath, 'utf8');
+    return { rel, keys: extractKvKeysFromEnvContent(content), readError: null };
+  } catch (e) {
+    return { rel, keys: [], readError: e };
+  }
+}
+
+function _scanBuilderEnvTemplates(catalog, pathsUtil) {
   const cwd = process.cwd();
   const fs = nodeFs();
+  const errors = [];
+  const scannedApps = [];
+  const scannedEnvTemplates = [];
+  const kvKeysUnique = new Set();
 
-  for (const { dir } of listAppDirsForDiscovery(pathsUtil)) {
+  for (const { dir } of listBuilderAppDirsForDiscovery(pathsUtil)) {
+    scannedApps.push(path.relative(cwd, dir) || dir);
     const envPath = path.join(dir, 'env.template');
     if (!fs.existsSync(envPath)) continue;
-    let content;
-    try {
-      content = fs.readFileSync(envPath, 'utf8');
-    } catch (e) {
-      errors.push(`Could not read ${envPath}: ${e.message}`);
+    const scan = _scanEnvTemplate(fs, cwd, envPath);
+    scannedEnvTemplates.push(scan.rel);
+    if (scan.readError) {
+      errors.push({
+        key: '__read_error__',
+        envTemplatePath: scan.rel,
+        message: scan.readError.message
+      });
       continue;
     }
-    const rel = path.relative(cwd, envPath) || envPath;
-    for (const k of extractKvKeysFromEnvContent(content)) {
+    for (const k of scan.keys) {
+      kvKeysUnique.add(k);
       if (!catalog.findEntryForKey(k)) {
-        errors.push(`Unknown kv:// key "${k}" in ${rel} — extend lib/schema/infra.parameter.yaml`);
+        errors.push({ key: k, envTemplatePath: scan.rel });
       }
     }
   }
 
-  return { valid: errors.length === 0, errors };
+  return { errors, scannedApps, scannedEnvTemplates, kvKeysUnique };
+}
+
+/**
+ * Validate that every kv:// key under builder app env.template files has catalog coverage.
+ * Integration apps under integration/ are not scanned (they often use ad-hoc kv keys).
+ * @param {{ findEntryForKey: Function }} catalog - Loaded catalog API
+ * @param {object} pathsUtil - paths module
+ * @returns {{
+ *   valid: boolean,
+ *   errors: { key: string, envTemplatePath: string, message?: string }[],
+ *   summary: {
+ *     scannedApps: string[],
+ *     scannedEnvTemplates: string[],
+ *     kvKeysUnique: string[],
+ *     kvKeysCount: number
+ *   }
+ * }}
+ */
+function validateWorkspaceKvRefsAgainstCatalog(catalog, pathsUtil) {
+  const { errors, scannedApps, scannedEnvTemplates, kvKeysUnique } = _scanBuilderEnvTemplates(
+    catalog,
+    pathsUtil
+  );
+
+  return {
+    valid: errors.length === 0,
+    errors,
+    summary: {
+      scannedApps: scannedApps.sort(),
+      scannedEnvTemplates: scannedEnvTemplates.sort(),
+      kvKeysUnique: [...kvKeysUnique].sort(),
+      kvKeysCount: kvKeysUnique.size
+    }
+  };
 }
 
 /**

package/lib/resolvers/datasource-resolver.js

@@ -0,0 +1,53 @@
+/**
+ * Shared datasource resolver utilities.
+ *
+ * Intentionally CLI-agnostic (no commander/options), so it can be reused by commands and tests.
+ *
+ * @fileoverview Datasource path + JSON loading helpers
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+const { resolveValidateInputPath, resolvePathFromIntegrationDatasourceKey } = require('../datasource/validate');
+
+/**
+ * @param {string} fileOrKey
+ * @returns {string} Absolute path to datasource JSON file
+ */
+function resolveDatasourceJsonPath(fileOrKey) {
+  return resolveValidateInputPath(String(fileOrKey || '').trim());
+}
+
+/**
+ * Attempt to resolve a datasource key under integration/<app>/ without throwing.
+ *
+ * @param {string} datasourceKey
+ * @returns {{ ok: true, path: string } | { ok: false, error: string }}
+ */
+function tryResolveDatasourceKeyToLocalPath(datasourceKey) {
+  try {
+    const p = resolvePathFromIntegrationDatasourceKey(String(datasourceKey || '').trim());
+    return { ok: true, path: p };
+  } catch (e) {
+    return { ok: false, error: e?.message || String(e) };
+  }
+}
+
+/**
+ * @param {string} jsonPath
+ * @returns {any}
+ */
+function readJsonFile(jsonPath) {
+  const raw = fs.readFileSync(jsonPath, 'utf8');
+  return JSON.parse(raw);
+}
+
+module.exports = {
+  resolveDatasourceJsonPath,
+  tryResolveDatasourceKeyToLocalPath,
+  readJsonFile
+};
+

package/lib/resolvers/dimension-file.js

@@ -0,0 +1,52 @@
+/**
+ * Dimension file helpers for `aifabrix dimension create --file`.
+ *
+ * @fileoverview Dimension file parsing
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * @typedef {Object} DimensionCreateInput
+ * @property {string} key
+ * @property {string} displayName
+ * @property {string} [description]
+ * @property {'string'|'number'|'boolean'} dataType
+ * @property {boolean} [isRequired]
+ * @property {Array<{ value: string, displayName?: string, description?: string }>} [values]
+ */
+
+/**
+ * @param {string} filePath
+ * @returns {DimensionCreateInput}
+ */
+function readDimensionCreateFile(filePath) {
+  const p = path.resolve(String(filePath || '').trim());
+  if (!p) {
+    throw new Error('--file is required');
+  }
+  if (!fs.existsSync(p)) {
+    throw new Error(`File not found: ${p}`);
+  }
+  const raw = fs.readFileSync(p, 'utf8');
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (e) {
+    throw new Error(`Invalid JSON in ${p}: ${e.message}`);
+  }
+  if (!parsed || typeof parsed !== 'object') {
+    throw new Error(`Dimension file must be a JSON object: ${p}`);
+  }
+  return parsed;
+}
+
+module.exports = {
+  readDimensionCreateFile
+};
+